diff --git a/backend/admission/apiserviceexportrequest_validator.go b/backend/admission/apiserviceexportrequest_validator.go new file mode 100644 index 000000000..d6b32f100 --- /dev/null +++ b/backend/admission/apiserviceexportrequest_validator.go @@ -0,0 +1,200 @@ +/* +Copyright 2025 The Kube Bind Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package admission + +import ( + "context" + "fmt" + "net/http" + "strings" + + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/klog/v2" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" + mcmanager "sigs.k8s.io/multicluster-runtime/pkg/manager" + + "github.com/kube-bind/kube-bind/backend/kubernetes/resources" + kubebindv1alpha2 "github.com/kube-bind/kube-bind/sdk/apis/kubebind/v1alpha2" + "github.com/kube-bind/kube-bind/sdk/apis/kubebind/v1alpha2/helpers" +) + +// APIServiceExportRequestValidator validates APIServiceExportRequest objects. +type APIServiceExportRequestValidator struct { + decoder admission.Decoder + manager mcmanager.Manager + informerScope kubebindv1alpha2.InformerScope + clusterScopedIsolation kubebindv1alpha2.Isolation + schemaSource string +} + +// NewAPIServiceExportRequestValidator creates a new validator for APIServiceExportRequest. +func NewAPIServiceExportRequestValidator( + manager mcmanager.Manager, + decoder admission.Decoder, + scope kubebindv1alpha2.InformerScope, + isolation kubebindv1alpha2.Isolation, + schemaSource string, +) *APIServiceExportRequestValidator { + return &APIServiceExportRequestValidator{ + decoder: decoder, + manager: manager, + informerScope: scope, + clusterScopedIsolation: isolation, + schemaSource: schemaSource, + } +} + +func (v *APIServiceExportRequestValidator) Handle(ctx context.Context, req admission.Request) admission.Response { + logger := log.FromContext(ctx) + ctx = klog.NewContext(ctx, logger) + + obj := &kubebindv1alpha2.APIServiceExportRequest{} + if err := v.decoder.Decode(req, obj); err != nil { + logger.Error(err, "Admission webhook: failed to decode APIServiceExportRequest") + return admission.Errored(http.StatusBadRequest, err) + } + + logger.Info("Admission webhook: decoded request", "resources", len(obj.Spec.Resources), "informerScope", v.informerScope) + + clusterName := "" + cl, err := v.manager.GetCluster(ctx, clusterName) + if err != nil { + clusterName = "default" + cl, err = v.manager.GetCluster(ctx, clusterName) + if err != nil { + logger.Info("Admission webhook: failed to get cluster for validation", "error", err) + return admission.Errored(http.StatusInternalServerError, fmt.Errorf("failed to get cluster: %w", err)) + } + } + client := cl.GetClient() + + if err := v.validateAPIServiceExportRequest(ctx, client, obj); err != nil { + return admission.Denied(err.Error()) + } + + logger.Info("Admission webhook: validation allowed") + return admission.Allowed("") +} + +func (v *APIServiceExportRequestValidator) validateAPIServiceExportRequest(ctx context.Context, cl client.Client, req *kubebindv1alpha2.APIServiceExportRequest) error { + logger := klog.FromContext(ctx) + logger.Info("Admission webhook: validating APIServiceExportRequest", "resources", len(req.Spec.Resources), "permissionClaims", len(req.Spec.PermissionClaims)) + + exportedSchemas, err := v.getExportedSchemas(ctx, cl) + if err != nil { + return err + } + + if len(exportedSchemas) == 0 { + return fmt.Errorf("no exported schemas found") + } + + first := apiextensionsv1.ResourceScope("") + for _, res := range req.Spec.Resources { + boundSchema, ok := exportedSchemas[res.ResourceGroupName()] + if !ok { + return fmt.Errorf("schema %s not found", res.ResourceGroupName()) + } + + if boundSchema.Spec.Scope == apiextensionsv1.ClusterScoped && v.informerScope != kubebindv1alpha2.ClusterScope { + return fmt.Errorf("resource %s/%s has scope %q which is incompatible with backend informer scope %q", res.Group, res.Resource, boundSchema.Spec.Scope, v.informerScope) + } + + if first == apiextensionsv1.ResourceScope("") { + first = boundSchema.Spec.Scope + continue + } + if boundSchema.Spec.Scope != first { + return fmt.Errorf("different scopes found for claimed resources: %v", boundSchema.Name) + } + } + + for _, claim := range req.Spec.PermissionClaims { + if !isClaimableAPI(claim) { + return fmt.Errorf("resource %s is not a valid claimable API", claim.GroupResource.String()) + } + } + + seenGroupResources := make(map[string]bool) + for _, claim := range req.Spec.PermissionClaims { + key := claim.Group + "/" + claim.Resource + if seenGroupResources[key] { + return fmt.Errorf("duplicate permission claim found for group/resource %s", claim.GroupResource.String()) + } + seenGroupResources[key] = true + } + + return nil +} + +func (v *APIServiceExportRequestValidator) getExportedSchemas(ctx context.Context, cl client.Client) (kubebindv1alpha2.ExportedSchemas, error) { + parts := strings.SplitN(v.schemaSource, ".", 3) + if len(parts) != 3 { + return nil, fmt.Errorf("malformed schema source: %q", v.schemaSource) + } + + gvk := schema.GroupVersionKind{ + Kind: parts[0], + Version: parts[1], + Group: parts[2], + } + + // Ensure we have the List kind + listGVK := gvk + if !strings.HasSuffix(listGVK.Kind, "List") { + listGVK.Kind += "List" + } + + list := &unstructured.UnstructuredList{} + list.SetGroupVersionKind(listGVK) + + labelSelector := labels.Set{ + resources.ExportedCRDsLabel: "true", + } + + listOpts := []client.ListOption{} + listOpts = append(listOpts, client.MatchingLabelsSelector{Selector: labelSelector.AsSelector()}) + + if err := cl.List(ctx, list, listOpts...); err != nil { + return nil, err + } + + boundSchemas := make(kubebindv1alpha2.ExportedSchemas, len(list.Items)) + for _, item := range list.Items { + boundSchema, err := helpers.UnstructuredToBoundSchema(item) + if err != nil { + return nil, err + } + boundSchemas[boundSchema.ResourceGroupName()] = boundSchema + } + + return boundSchemas, nil +} + +func isClaimableAPI(claim kubebindv1alpha2.PermissionClaim) bool { + for _, api := range kubebindv1alpha2.ClaimableAPIs { + if claim.Group == api.GroupVersionResource.Group && claim.Resource == api.Names.Plural { + return true + } + } + return false +} diff --git a/backend/config.go b/backend/config.go index e0a0cce26..a0cb1ee69 100644 --- a/backend/config.go +++ b/backend/config.go @@ -25,20 +25,26 @@ import ( apisv1alpha1 "github.com/kcp-dev/kcp/sdk/apis/apis/v1alpha1" apisv1alpha2 "github.com/kcp-dev/kcp/sdk/apis/apis/v1alpha2" "github.com/kcp-dev/multicluster-provider/apiexport" + admissionregistrationv1 "k8s.io/api/admissionregistration/v1" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/kubernetes" clientgoscheme "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" + "k8s.io/klog/v2" "k8s.io/utils/ptr" ctrl "sigs.k8s.io/controller-runtime" + ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" ctrlconfig "sigs.k8s.io/controller-runtime/pkg/config" metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" + "sigs.k8s.io/controller-runtime/pkg/webhook" mcmanager "sigs.k8s.io/multicluster-runtime/pkg/manager" "sigs.k8s.io/multicluster-runtime/pkg/multicluster" kuberesources "github.com/kube-bind/kube-bind/backend/kubernetes/resources" "github.com/kube-bind/kube-bind/backend/options" + webhookpkg "github.com/kube-bind/kube-bind/backend/webhook" kubebindv1alpha1 "github.com/kube-bind/kube-bind/sdk/apis/kubebind/v1alpha1" kubebindv1alpha2 "github.com/kube-bind/kube-bind/sdk/apis/kubebind/v1alpha2" ) @@ -82,6 +88,9 @@ func NewConfig(options *options.CompletedOptions) (*Config, error) { if err := apiextensionsv1.AddToScheme(scheme); err != nil { return nil, fmt.Errorf("error adding apiextensions scheme: %w", err) } + if err := admissionregistrationv1.AddToScheme(scheme); err != nil { + return nil, fmt.Errorf("error adding admissionregistration scheme: %w", err) + } if err := kubebindv1alpha1.AddToScheme(scheme); err != nil { return nil, fmt.Errorf("error adding kubebind scheme: %w", err) } @@ -118,6 +127,34 @@ func NewConfig(options *options.CompletedOptions) (*Config, error) { config.Provider = nil } + // Try to generate certificates using cert-manager + ctx := context.Background() + logger := klog.FromContext(ctx) + + kubeClient, err := kubernetes.NewForConfig(config.ClientConfig) + if err == nil { + if crClient, err := ctrlclient.New(config.ClientConfig, ctrlclient.Options{Scheme: scheme}); err == nil { + if err := webhookpkg.EnsureWebhookCertificates(ctx, config.ClientConfig, kubeClient, crClient, scheme); err != nil { + logger.V(2).Info("Could not generate certificates via cert-manager", "error", err) + } + } + + hasCertManager, err := webhookpkg.CheckCertManagerInstalled(ctx, config.ClientConfig) + if err == nil && hasCertManager { + webhookpkg.StartWebhookCertificateWatcher(ctx, kubeClient) + logger.V(1).Info("Started webhook certificate watcher for automatic rotation") + } + } else { + logger.V(1).Info("Failed to create kubeClient for webhook certificates", "error", err) + } + + webhookServer := webhook.NewServer(webhook.Options{ + Port: options.WebhookPort, + CertDir: webhookpkg.WebhookCertDirectory, + }) + + logger.V(1).Info("Webhook server enabled with certificates", "certDir", webhookpkg.WebhookCertDirectory) + opts := ctrl.Options{ Controller: ctrlconfig.Controller{ SkipNameValidation: ptr.To(config.Options.ExtraOptions.TestingSkipNameValidation), @@ -125,7 +162,8 @@ func NewConfig(options *options.CompletedOptions) (*Config, error) { Metrics: metricsserver.Options{ BindAddress: "0", }, - Scheme: scheme, + Scheme: scheme, + WebhookServer: webhookServer, } manager, err := mcmanager.New(config.ClientConfig, config.Provider, opts) diff --git a/backend/controllers/serviceexportrequest/serviceexportrequest_reconcile.go b/backend/controllers/serviceexportrequest/serviceexportrequest_reconcile.go index 59c3584ef..380a3b4d7 100644 --- a/backend/controllers/serviceexportrequest/serviceexportrequest_reconcile.go +++ b/backend/controllers/serviceexportrequest/serviceexportrequest_reconcile.go @@ -53,18 +53,11 @@ type reconciler struct { } func (r *reconciler) reconcile(ctx context.Context, cl client.Client, cache cache.Cache, req *kubebindv1alpha2.APIServiceExportRequest) error { - // We must ensure schemas are created in form of boundSchemas first for the validation. - // Worst case scenario if validation fails, we will reuse schemas for same consumer once issues are fixed. if err := r.ensureBoundSchemas(ctx, cl, cache, req); err != nil { conditions.SetSummary(req) return fmt.Errorf("failed to ensure bound schemas: %w", err) } - if err := r.validate(ctx, cl, req); err != nil { - conditions.SetSummary(req) - return fmt.Errorf("failed to validate APIServiceExportRequest: %w", err) - } - if err := r.ensureExports(ctx, cl, cache, req); err != nil { conditions.SetSummary(req) return fmt.Errorf("failed to ensure exports: %w", err) @@ -270,109 +263,6 @@ func (r *reconciler) ensureExports(ctx context.Context, cl client.Client, cache return nil } -// Validate validates if the APIServiceExportRequest is in a valid state. -// Currently it validates if all requested schemas are of the same scope and -// if claimable apis are allowed and valid. -// -// TODO: Move this to validatingAdmissionWebhook as this is not really part of reconciliation. -// https://github.com/kube-bind/kube-bind/issues/325 -func (r *reconciler) validate(ctx context.Context, cl client.Client, req *kubebindv1alpha2.APIServiceExportRequest) error { - exportedSchemas, err := r.getExportedSchemas(ctx, cl) - if err != nil { - return err - } - - if len(exportedSchemas) == 0 { - conditions.MarkFalse( - req, - kubebindv1alpha2.APIServiceExportRequestConditionExportsReady, - "SchemaNotFound", - conditionsapi.ConditionSeverityError, - "Schema not found", - ) - return fmt.Errorf("no exported schemas found") - } - - first := apiextensionsv1.ResourceScope("") - for _, res := range req.Spec.Resources { - boundSchema, ok := exportedSchemas[res.ResourceGroupName()] - if !ok { - conditions.MarkFalse( - req, - kubebindv1alpha2.APIServiceExportRequestConditionExportsReady, - "SchemaNotFound", - conditionsapi.ConditionSeverityError, - "Schema %s not found", - res.ResourceGroupName(), - ) - return fmt.Errorf("schema %s not found", res.ResourceGroupName()) - } - if first == apiextensionsv1.ResourceScope("") { - first = boundSchema.Spec.Scope - continue - } - if boundSchema.Spec.Scope != first { - conditions.MarkFalse(req, - kubebindv1alpha2.APIServiceExportRequestConditionExportsReady, - "DifferentScopes", - conditionsapi.ConditionSeverityError, - "Different scopes found: %v", - boundSchema.Spec.Scope, - ) - return fmt.Errorf("different scopes found for claimed resources: %v", boundSchema.Name) - } - } - - // Add validation if claimable apis are valid here - for _, claim := range req.Spec.PermissionClaims { - if !isClaimableAPI(claim) { - conditions.MarkFalse( - req, - kubebindv1alpha2.APIServiceExportConditionPermissionClaim, - "InvalidPermissionClaim", - conditionsapi.ConditionSeverityError, - "Resource %s is not a valid claimable API", - claim.GroupResource.String(), - ) - req.Status.Phase = kubebindv1alpha2.APIServiceExportRequestPhaseFailed - req.Status.TerminalMessage = conditions.GetMessage(req, kubebindv1alpha2.APIServiceExportConditionPermissionClaim) - return fmt.Errorf("resource %s is not a valid claimable API", claim.GroupResource.String()) - } - } - - // Add validation for duplicate group/resource combinations - seenGroupResources := make(map[string]bool) - for _, claim := range req.Spec.PermissionClaims { - key := claim.Group + "/" + claim.Resource - if seenGroupResources[key] { - conditions.MarkFalse( - req, - kubebindv1alpha2.APIServiceExportConditionPermissionClaim, - "DuplicatePermissionClaim", - conditionsapi.ConditionSeverityError, - "Duplicate permission claim found for group/resource %s", - claim.GroupResource.String(), - ) - req.Status.Phase = kubebindv1alpha2.APIServiceExportRequestPhaseFailed - req.Status.TerminalMessage = conditions.GetMessage(req, kubebindv1alpha2.APIServiceExportConditionPermissionClaim) - return fmt.Errorf("duplicate permission claim found for group/resource %s", claim.GroupResource.String()) - } - seenGroupResources[key] = true - } - - return nil -} - -// isClaimableAPI checks if a permission claim is for a claimable API. -func isClaimableAPI(claim kubebindv1alpha2.PermissionClaim) bool { - for _, api := range kubebindv1alpha2.ClaimableAPIs { - if claim.Group == api.GroupVersionResource.Group && claim.Resource == api.Names.Plural { - return true - } - } - return false -} - func (r *reconciler) ensureAPIServiceNamespaces(ctx context.Context, cl client.Client, cache cache.Cache, req *kubebindv1alpha2.APIServiceExportRequest) error { logger := klog.FromContext(ctx) diff --git a/backend/options/options.go b/backend/options/options.go index d4d1eb1cf..b9e8a20aa 100644 --- a/backend/options/options.go +++ b/backend/options/options.go @@ -53,6 +53,7 @@ type ExtraOptions struct { ExternalCAFile string ExternalCA []byte TLSExternalServerName string + WebhookPort int // Defines the source of the schema for the bind screen. // Options are: // CustomResourceDefinition.v1.apiextensions.k8s.io @@ -100,6 +101,7 @@ func NewOptions() *Options { ServerURL: "", SchemaSource: CustomResourceDefinitionSource.String(), Frontend: "embedded", // Not used, but indicates to use embedded frontend using SPA. + WebhookPort: 9443, }, } } @@ -144,6 +146,7 @@ func (options *Options) AddFlags(fs *pflag.FlagSet) { fs.StringVar(&options.ExternalAddress, "external-address", options.ExternalAddress, "The external address for the service provider cluster, including https:// and port. If not specified, service account's hosts are used.") fs.StringVar(&options.ExternalCAFile, "external-ca-file", options.ExternalCAFile, "The external CA file for the service provider cluster. If not specified, service account's CA is used.") fs.StringVar(&options.TLSExternalServerName, "external-server-name", options.TLSExternalServerName, "The external (TLS) server name used by consumers to talk to the service provider cluster. This can be useful to select the right certificate via SNI.") + fs.IntVar(&options.WebhookPort, "webhook-port", options.WebhookPort, "The port on which the webhook server listens. Defaults to 9443.") fs.StringVar(&options.Frontend, "frontend", options.Frontend, "If starts with http:// it is treated as a URL to a SPA server Else - it is treated as a path to static files to be served.") fs.StringVar(&options.Provider, "multicluster-runtime-provider", options.Provider, diff --git a/backend/server.go b/backend/server.go index d17e985f4..b13004462 100644 --- a/backend/server.go +++ b/backend/server.go @@ -23,11 +23,15 @@ import ( "net" "sync" + "k8s.io/client-go/kubernetes" "k8s.io/klog/v2" "k8s.io/utils/ptr" + ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" mcreconcile "sigs.k8s.io/multicluster-runtime/pkg/reconcile" + admissionpkg "github.com/kube-bind/kube-bind/backend/admission" "github.com/kube-bind/kube-bind/backend/auth" "github.com/kube-bind/kube-bind/backend/controllers/clusterbinding" "github.com/kube-bind/kube-bind/backend/controllers/serviceexport" @@ -35,6 +39,7 @@ import ( "github.com/kube-bind/kube-bind/backend/controllers/servicenamespace" http "github.com/kube-bind/kube-bind/backend/http" kube "github.com/kube-bind/kube-bind/backend/kubernetes" + webhookpkg "github.com/kube-bind/kube-bind/backend/webhook" kubebindv1alpha2 "github.com/kube-bind/kube-bind/sdk/apis/kubebind/v1alpha2" ) @@ -202,6 +207,10 @@ func NewServer(ctx context.Context, c *Config) (*Server, error) { return nil, fmt.Errorf("error setting up ServiceExportRequest controller with manager: %w", err) } + if err := registerAPIServiceExportRequestWebhook(ctx, s.Config); err != nil { + return nil, fmt.Errorf("error setting up APIServiceExportRequest webhook: %w", err) + } + return s, nil } @@ -255,6 +264,54 @@ func (s *Server) GetOIDCProvider(ctx context.Context) (*auth.OIDCServiceProvider return s.OIDC, nil } +func registerAPIServiceExportRequestWebhook(ctx context.Context, c *Config) error { + webhookServer := c.Manager.GetWebhookServer() + + decoder := admission.NewDecoder(c.Scheme) + + validator := admissionpkg.NewAPIServiceExportRequestValidator( + c.Manager, + decoder, + kubebindv1alpha2.InformerScope(c.Options.ConsumerScope), + kubebindv1alpha2.Isolation(c.Options.ClusterScopedIsolation), + c.Options.SchemaSource, + ) + + webhookServer.Register(webhookpkg.WebhookPath, &admission.Webhook{ + Handler: validator, + }) + + logger := klog.FromContext(ctx) + kubeClient, err := kubernetes.NewForConfig(c.ClientConfig) + if err != nil { + return fmt.Errorf("failed to create kubernetes client: %w", err) + } + + crClient, err := ctrlclient.New(c.ClientConfig, ctrlclient.Options{Scheme: c.Scheme}) + if err != nil { + return fmt.Errorf("failed to create controller-runtime client: %w", err) + } + + webhookURL, err := webhookpkg.GetWebhookURL(ctx, c.ClientConfig, c.Options.WebhookPort) + if err != nil { + return fmt.Errorf("failed to determine webhook URL: %w", err) + } + logger.V(1).Info("Using webhook URL from kubeconfig", "url", webhookURL) + + if err := webhookpkg.EnsureValidatingWebhookConfiguration( + ctx, + crClient, + kubeClient, + webhookURL, + webhookpkg.WebhookCertDirectory, + ); err != nil { + logger.V(1).Info("Failed to create ValidatingWebhookConfiguration", "error", err) + return err + } + + return nil +} + func (s *Server) Addr() net.Addr { return s.WebServer.Addr() } diff --git a/backend/webhook/certificate.go b/backend/webhook/certificate.go new file mode 100644 index 000000000..6927ffb8f --- /dev/null +++ b/backend/webhook/certificate.go @@ -0,0 +1,219 @@ +/* +Copyright 2025 The Kube Bind Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package webhook + +import ( + "context" + "crypto/rand" + "crypto/rsa" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "fmt" + "time" + + certificatesv1 "k8s.io/api/certificates/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "k8s.io/klog/v2" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +const ( + webhookCSRName = "kube-bind-webhook" +) + +// GenerateWebhookCertificate generates a certificate for the webhook server using +// Kubernetes Certificate Signing Request API. It creates a CSR, waits for approval, +// and returns a TLS certificate that can be used for the webhook server. +func GenerateWebhookCertificate(ctx context.Context, clientConfig *rest.Config, kubeClient kubernetes.Interface, crClient client.Client, commonName string) (*tls.Certificate, error) { + logger := klog.FromContext(ctx) + + privateKey, err := rsa.GenerateKey(rand.Reader, 2048) + if err != nil { + return nil, fmt.Errorf("failed to generate private key: %w", err) + } + + csrBytes, err := createCertificateRequest(privateKey, commonName) + if err != nil { + return nil, fmt.Errorf("failed to create certificate request: %w", err) + } + + csr := &certificatesv1.CertificateSigningRequest{ + ObjectMeta: metav1.ObjectMeta{ + Name: webhookCSRName, + }, + Spec: certificatesv1.CertificateSigningRequestSpec{ + Request: csrBytes, + SignerName: "kubernetes.io/kubelet-serving", + Usages: []certificatesv1.KeyUsage{ + certificatesv1.UsageDigitalSignature, + certificatesv1.UsageKeyEncipherment, + certificatesv1.UsageServerAuth, + }, + }, + } + + var existingCSR certificatesv1.CertificateSigningRequest + err = crClient.Get(ctx, types.NamespacedName{Name: webhookCSRName}, &existingCSR) + if err == nil { + if len(existingCSR.Status.Certificate) > 0 { + logger.V(2).Info("Found existing approved CSR certificate") + logger.V(2).Info("Creating new CSR as private key cannot be recovered from existing CSR") + if err := crClient.Delete(ctx, &existingCSR); err != nil { + logger.V(1).Info("Failed to delete existing CSR, continuing", "error", err) + } + } else { + logger.V(2).Info("Deleting existing unapproved CSR to create new one") + if err := crClient.Delete(ctx, &existingCSR); err != nil { + logger.V(1).Info("Failed to delete existing CSR, continuing", "error", err) + } + } + } + + logger.V(1).Info("Creating CertificateSigningRequest for webhook server") + if err := crClient.Create(ctx, csr); err != nil { + return nil, fmt.Errorf("failed to create CSR: %w", err) + } + + // Auto-approve the CSR if we have permissions + if err := autoApproveCSR(ctx, kubeClient, webhookCSRName); err != nil { + logger.V(1).Info("Failed to auto-approve CSR, you may need to approve it manually", "error", err) + logger.V(1).Info("To approve manually, run: kubectl certificate approve " + webhookCSRName) + } + + logger.V(1).Info("Waiting for CSR to be approved and certificate to be issued") + var certBytes []byte + if err := wait.PollUntilContextTimeout(ctx, 1*time.Second, 30*time.Second, true, func(ctx context.Context) (done bool, err error) { + var csrObj certificatesv1.CertificateSigningRequest + if err := crClient.Get(ctx, types.NamespacedName{Name: webhookCSRName}, &csrObj); err != nil { + return false, err + } + + approved := false + for _, condition := range csrObj.Status.Conditions { + if condition.Type == certificatesv1.CertificateApproved && condition.Status == corev1.ConditionTrue { + approved = true + break + } + } + + if approved && len(csrObj.Status.Certificate) > 0 { + certBytes = csrObj.Status.Certificate + return true, nil + } + + return false, nil + }); err != nil { + return nil, fmt.Errorf("failed to get certificate from CSR: %w", err) + } + + return parseCertificate(certBytes, privateKey) +} + +func createCertificateRequest(privateKey *rsa.PrivateKey, commonName string) ([]byte, error) { + template := &x509.CertificateRequest{ + Subject: pkix.Name{ + CommonName: commonName, + }, + } + + csrBytes, err := x509.CreateCertificateRequest(rand.Reader, template, privateKey) + if err != nil { + return nil, fmt.Errorf("failed to create certificate request: %w", err) + } + + csrPEM := pem.EncodeToMemory(&pem.Block{ + Type: "CERTIFICATE REQUEST", + Bytes: csrBytes, + }) + + return csrPEM, nil +} + +func parseCertificate(certBytes []byte, privateKey *rsa.PrivateKey) (*tls.Certificate, error) { + block, _ := pem.Decode(certBytes) + if block == nil { + return nil, fmt.Errorf("failed to decode certificate PEM") + } + + cert, err := x509.ParseCertificate(block.Bytes) + if err != nil { + return nil, fmt.Errorf("failed to parse certificate: %w", err) + } + + return &tls.Certificate{ + Certificate: [][]byte{cert.Raw}, + PrivateKey: privateKey, + Leaf: cert, + }, nil +} + +func autoApproveCSR(ctx context.Context, kubeClient kubernetes.Interface, csrName string) error { + csr, err := kubeClient.CertificatesV1().CertificateSigningRequests().Get(ctx, csrName, metav1.GetOptions{}) + if err != nil { + return err + } + + for _, condition := range csr.Status.Conditions { + if condition.Type == certificatesv1.CertificateApproved { + return nil + } + } + + approval := certificatesv1.CertificateSigningRequest{ + ObjectMeta: metav1.ObjectMeta{ + Name: csrName, + }, + Status: certificatesv1.CertificateSigningRequestStatus{ + Conditions: []certificatesv1.CertificateSigningRequestCondition{ + { + Type: certificatesv1.CertificateApproved, + Status: corev1.ConditionTrue, + Reason: "AutoApproved", + Message: "Auto-approved by kube-bind backend", + }, + }, + }, + } + + _, err = kubeClient.CertificatesV1().CertificateSigningRequests().UpdateApproval(ctx, csrName, &approval, metav1.UpdateOptions{}) + return err +} + +func GetTLSOptionsForWebhook(ctx context.Context, clientConfig *rest.Config, kubeClient kubernetes.Interface, crClient client.Client) ([]func(*tls.Config), error) { + commonName := "kube-bind-webhook" + + cert, err := GenerateWebhookCertificate(ctx, clientConfig, kubeClient, crClient, commonName) + if err != nil { + klog.FromContext(ctx).V(1).Info("Failed to generate certificate via CSR, will use file-based certs", "error", err) + return nil, nil + } + + return []func(*tls.Config){ + func(cfg *tls.Config) { + cfg.GetCertificate = func(*tls.ClientHelloInfo) (*tls.Certificate, error) { + return cert, nil + } + }, + }, nil +} diff --git a/backend/webhook/certmanager.go b/backend/webhook/certmanager.go new file mode 100644 index 000000000..bc928c0d2 --- /dev/null +++ b/backend/webhook/certmanager.go @@ -0,0 +1,449 @@ +/* +Copyright 2025 The Kube Bind Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package webhook + +import ( + "context" + "crypto/rand" + "crypto/rsa" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "fmt" + "math/big" + "os" + "path/filepath" + "time" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/fields" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/dynamic" + "k8s.io/client-go/informers" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/cache" + "k8s.io/klog/v2" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +const ( + certManagerGroup = "cert-manager.io" + certManagerVersion = "v1" + certManagerKind = "Certificate" + issuerKind = "Issuer" + + webhookCertName = "kube-bind-webhook-cert" + webhookCertSecret = "kube-bind-webhook-cert" //nolint:gosec // This is a secret name, not credentials + webhookIssuerName = "kube-bind-webhook-issuer" + webhookCertNamespace = "kube-bind" +) + +// EnsureWebhookCertificates uses cert-manager to generate certificates or falls back to self-signed +func EnsureWebhookCertificates(ctx context.Context, cfg *rest.Config, kubeClient kubernetes.Interface, crClient client.Client, scheme *runtime.Scheme) error { + logger := klog.FromContext(ctx) + + certManagerErr := ensureCertsViaCertManager(ctx, cfg, kubeClient, crClient, scheme) + if certManagerErr == nil { + return nil + } + + logger.V(1).Info("Cert-manager path failed, trying fallback", "error", certManagerErr) + + if err := generateSelfSignedWebhookCertificates(WebhookCertDirectory); err != nil { + return fmt.Errorf("failed to generate fallback webhook certificates: %w", err) + } + + logger.V(1).Info("Generated fallback self-signed webhook certificates", "certDir", WebhookCertDirectory) + + return nil +} + +func ensureCertsViaCertManager(ctx context.Context, cfg *rest.Config, kubeClient kubernetes.Interface, crClient client.Client, scheme *runtime.Scheme) error { + logger := klog.FromContext(ctx) + + hasCertManager, err := CheckCertManagerInstalled(ctx, cfg) + if err != nil { + logger.V(2).Info("Error checking cert-manager installation", "error", err) + return err + } + if !hasCertManager { + logger.V(2).Info("Cert-manager not installed, skipping certificate generation") + return fmt.Errorf("cert-manager not installed") + } + + logger.V(1).Info("Cert-manager detected, generating webhook certificates") + + ns := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: webhookCertNamespace, + }, + } + if err := crClient.Get(ctx, types.NamespacedName{Name: webhookCertNamespace}, ns); err != nil { + if err := crClient.Create(ctx, ns); err != nil { + logger.V(1).Info("Failed to create namespace, may already exist", "error", err) + } + } + + if err := createSelfSignedIssuer(ctx, cfg, crClient); err != nil { + return fmt.Errorf("failed to create issuer: %w", err) + } + + if err := createCertificate(ctx, cfg, crClient); err != nil { + return fmt.Errorf("failed to create certificate: %w", err) + } + + logger.V(1).Info("Waiting for certificate to be ready") + if err := waitForCertificateReady(ctx, cfg); err != nil { + return fmt.Errorf("certificate not ready: %w", err) + } + + if err := extractAndWriteCertificates(ctx, kubeClient); err != nil { + return fmt.Errorf("failed to extract certificates: %w", err) + } + + logger.V(1).Info("Successfully generated webhook certificates", "certDir", WebhookCertDirectory) + + return nil +} + +// CheckCertManagerInstalled checks if cert-manager is installed in the cluster +func CheckCertManagerInstalled(ctx context.Context, cfg *rest.Config) (bool, error) { + dynamicClient, err := dynamic.NewForConfig(cfg) + if err != nil { + return false, err + } + + certGVR := schema.GroupVersionResource{ + Group: certManagerGroup, + Version: certManagerVersion, + Resource: "certificates", + } + + _, err = dynamicClient.Resource(certGVR).List(ctx, metav1.ListOptions{Limit: 1}) + if err != nil { + return false, err + } + + return true, nil +} + +func createSelfSignedIssuer(ctx context.Context, cfg *rest.Config, crClient client.Client) error { + logger := klog.FromContext(ctx) + + dynamicClient, err := dynamic.NewForConfig(cfg) + if err != nil { + return err + } + + issuerGVR := schema.GroupVersionResource{ + Group: certManagerGroup, + Version: certManagerVersion, + Resource: "issuers", + } + + _, err = dynamicClient.Resource(issuerGVR).Namespace(webhookCertNamespace).Get(ctx, webhookIssuerName, metav1.GetOptions{}) + if err == nil { + logger.V(2).Info("Issuer already exists") + return nil + } + + issuer := map[string]interface{}{ + "apiVersion": fmt.Sprintf("%s/%s", certManagerGroup, certManagerVersion), + "kind": issuerKind, + "metadata": map[string]interface{}{ + "name": webhookIssuerName, + "namespace": webhookCertNamespace, + }, + "spec": map[string]interface{}{ + "selfSigned": map[string]interface{}{}, + }, + } + + unstructuredObj := &unstructured.Unstructured{Object: issuer} + if err := crClient.Create(ctx, unstructuredObj); err != nil { + return fmt.Errorf("failed to create issuer: %w", err) + } + + logger.V(1).Info("Created self-signed issuer") + return nil +} + +func createCertificate(ctx context.Context, cfg *rest.Config, crClient client.Client) error { + logger := klog.FromContext(ctx) + + dynamicClient, err := dynamic.NewForConfig(cfg) + if err != nil { + return err + } + + certGVR := schema.GroupVersionResource{ + Group: certManagerGroup, + Version: certManagerVersion, + Resource: "certificates", + } + + _, err = dynamicClient.Resource(certGVR).Namespace(webhookCertNamespace).Get(ctx, webhookCertName, metav1.GetOptions{}) + if err == nil { + return nil + } + + cert := map[string]interface{}{ + "apiVersion": fmt.Sprintf("%s/%s", certManagerGroup, certManagerVersion), + "kind": certManagerKind, + "metadata": map[string]interface{}{ + "name": webhookCertName, + "namespace": webhookCertNamespace, + }, + "spec": map[string]interface{}{ + "secretName": webhookCertSecret, + "issuerRef": map[string]interface{}{ + "name": webhookIssuerName, + "kind": issuerKind, + }, + "commonName": "kube-bind-webhook", + }, + } + + unstructuredObj := &unstructured.Unstructured{Object: cert} + if err := crClient.Create(ctx, unstructuredObj); err != nil { + return fmt.Errorf("failed to create certificate: %w", err) + } + + logger.V(1).Info("Created certificate resource") + return nil +} + +func waitForCertificateReady(ctx context.Context, cfg *rest.Config) error { + dynamicClient, err := dynamic.NewForConfig(cfg) + if err != nil { + return err + } + + certGVR := schema.GroupVersionResource{ + Group: certManagerGroup, + Version: certManagerVersion, + Resource: "certificates", + } + + return wait.PollUntilContextTimeout(ctx, 1*time.Second, 60*time.Second, true, func(ctx context.Context) (done bool, err error) { + cert, err := dynamicClient.Resource(certGVR).Namespace(webhookCertNamespace).Get(ctx, webhookCertName, metav1.GetOptions{}) + if err != nil { + return false, err + } + + status, found, err := unstructured.NestedMap(cert.Object, "status") + if err != nil { + return false, err + } + if !found { + return false, nil + } + + conditions, found, err := unstructured.NestedSlice(status, "conditions") + if err != nil { + return false, err + } + if !found { + return false, nil + } + + for _, cond := range conditions { + condMap, ok := cond.(map[string]interface{}) + if !ok { + continue + } + if condMap["type"] == "Ready" && condMap["status"] == "True" { + return true, nil + } + } + + return false, nil + }) +} + +func extractAndWriteCertificates(ctx context.Context, kubeClient kubernetes.Interface) error { + secret, err := kubeClient.CoreV1().Secrets(webhookCertNamespace).Get(ctx, webhookCertSecret, metav1.GetOptions{}) + if err != nil { + return fmt.Errorf("failed to get secret: %w", err) + } + return writeSecretToFiles(secret) +} + +func writeSecretToFiles(secret *corev1.Secret) error { + if err := os.MkdirAll(WebhookCertDirectory, 0755); err != nil { + return fmt.Errorf("failed to create cert directory: %w", err) + } + + certPath := filepath.Join(WebhookCertDirectory, "tls.crt") + keyPath := filepath.Join(WebhookCertDirectory, "tls.key") + + needsUpdate, err := shouldUpdateCertificate(certPath, secret.Data["tls.crt"]) + if err != nil { + needsUpdate = true + } + + if !needsUpdate { + return nil + } + + if certData, exists := secret.Data["tls.crt"]; exists { + if err := os.WriteFile(certPath, certData, 0600); err != nil { + return fmt.Errorf("failed to write certificate: %w", err) + } + } else { + return fmt.Errorf("tls.crt not found in secret") + } + + if keyData, exists := secret.Data["tls.key"]; exists { + if err := os.WriteFile(keyPath, keyData, 0600); err != nil { + return fmt.Errorf("failed to write key: %w", err) + } + } else { + return fmt.Errorf("tls.key not found in secret") + } + + return nil +} + +func shouldUpdateCertificate(certPath string, newCertData []byte) (bool, error) { + existingCertData, err := os.ReadFile(certPath) + if err != nil { + if os.IsNotExist(err) { + return true, nil + } + return false, fmt.Errorf("failed to read existing certificate: %w", err) + } + + if len(existingCertData) != len(newCertData) { + return true, nil + } + for i := range existingCertData { + if existingCertData[i] != newCertData[i] { + return true, nil + } + } + + block, _ := pem.Decode(existingCertData) + if block == nil { + return true, nil + } + + cert, err := x509.ParseCertificate(block.Bytes) + if err != nil { + return true, err + } + + now := time.Now() + renewalThreshold := now.Add(7 * 24 * time.Hour) + if cert.NotAfter.Before(renewalThreshold) { + return true, nil + } + + return false, nil +} + +func generateSelfSignedWebhookCertificates(certDir string) error { + if err := os.MkdirAll(certDir, 0o755); err != nil { + return fmt.Errorf("failed to create cert directory: %w", err) + } + + privateKey, err := rsa.GenerateKey(rand.Reader, 2048) + if err != nil { + return fmt.Errorf("failed to generate private key: %w", err) + } + + serialNumberLimit := new(big.Int).Lsh(big.NewInt(1), 128) + serialNumber, err := rand.Int(rand.Reader, serialNumberLimit) + if err != nil { + return fmt.Errorf("failed to generate serial number: %w", err) + } + + template := &x509.Certificate{ + SerialNumber: serialNumber, + Subject: pkix.Name{ + CommonName: "kube-bind-webhook", + }, + NotBefore: time.Now().Add(-time.Hour), + NotAfter: time.Now().Add(365 * 24 * time.Hour), + KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment | x509.KeyUsageCertSign, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + BasicConstraintsValid: true, + IsCA: true, + } + + certDER, err := x509.CreateCertificate(rand.Reader, template, template, &privateKey.PublicKey, privateKey) + if err != nil { + return fmt.Errorf("failed to create certificate: %w", err) + } + + certPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: certDER}) + keyPEM := pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(privateKey)}) + + if err := os.WriteFile(filepath.Join(certDir, "tls.crt"), certPEM, 0o600); err != nil { + return fmt.Errorf("failed to write certificate: %w", err) + } + if err := os.WriteFile(filepath.Join(certDir, "tls.key"), keyPEM, 0o600); err != nil { + return fmt.Errorf("failed to write key: %w", err) + } + + return nil +} + +// StartWebhookCertificateWatcher watches the cert-manager secret and keeps the local files in sync. +func StartWebhookCertificateWatcher(ctx context.Context, kubeClient kubernetes.Interface) { + logger := klog.FromContext(ctx) + + factory := informers.NewSharedInformerFactoryWithOptions( + kubeClient, + 30*time.Second, + informers.WithNamespace(webhookCertNamespace), + informers.WithTweakListOptions(func(opts *metav1.ListOptions) { + opts.FieldSelector = fields.OneTermEqualSelector("metadata.name", webhookCertSecret).String() + }), + ) + + informer := factory.Core().V1().Secrets().Informer() + + handleSecret := func(obj interface{}) { + secret, ok := obj.(*corev1.Secret) + if !ok { + return + } + if secret.Name != webhookCertSecret { + return + } + + if err := writeSecretToFiles(secret); err != nil { + logger.Error(err, "Failed to sync webhook certificate secret to disk") + return + } + } + + //nolint:errcheck // AddEventHandler doesn't return an error + informer.AddEventHandler(cache.ResourceEventHandlerFuncs{ + AddFunc: handleSecret, + UpdateFunc: func(_, newObj interface{}) { handleSecret(newObj) }, + }) + + go informer.Run(ctx.Done()) +} diff --git a/backend/webhook/webhookconfig.go b/backend/webhook/webhookconfig.go new file mode 100644 index 000000000..98968f1fb --- /dev/null +++ b/backend/webhook/webhookconfig.go @@ -0,0 +1,261 @@ +/* +Copyright 2025 The Kube Bind Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package webhook + +import ( + "context" + "crypto/x509" + "encoding/pem" + "fmt" + "net" + "net/url" + "os" + "path/filepath" + + admissionregistrationv1 "k8s.io/api/admissionregistration/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "k8s.io/klog/v2" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +const ( + validatingWebhookConfigName = "kube-bind-apiserviceexportrequest-validating-webhook" + WebhookPath = "/validate-apiserviceexportrequest" + WebhookCertDirectory = "/tmp/k8s-webhook-server/serving-certs" +) + +func EnsureValidatingWebhookConfiguration(ctx context.Context, crClient client.Client, kubeClient kubernetes.Interface, webhookURL string, certDir string) error { + logger := klog.FromContext(ctx) + + _, err := url.Parse(webhookURL) + if err != nil { + return fmt.Errorf("failed to parse webhook URL: %w", err) + } + + certPath := filepath.Join(certDir, "tls.crt") + certData, err := os.ReadFile(certPath) + if err != nil { + return fmt.Errorf("failed to read certificate: %w", err) + } + + caCert, err := extractCABundle(certData) + if err != nil { + return fmt.Errorf("failed to extract CA bundle: %w", err) + } + + var existing admissionregistrationv1.ValidatingWebhookConfiguration + err = crClient.Get(ctx, types.NamespacedName{Name: validatingWebhookConfigName}, &existing) + if err == nil { + logger.V(1).Info("Updating existing ValidatingWebhookConfiguration") + existing.Webhooks = []admissionregistrationv1.ValidatingWebhook{ + buildValidatingWebhook(webhookURL, caCert), + } + if err := crClient.Update(ctx, &existing); err != nil { + return fmt.Errorf("failed to update ValidatingWebhookConfiguration: %w", err) + } + logger.V(1).Info("Updated ValidatingWebhookConfiguration") + return nil + } + + webhookConfig := &admissionregistrationv1.ValidatingWebhookConfiguration{ + ObjectMeta: metav1.ObjectMeta{ + Name: validatingWebhookConfigName, + }, + Webhooks: []admissionregistrationv1.ValidatingWebhook{ + buildValidatingWebhook(webhookURL, caCert), + }, + } + + logger.V(1).Info("Creating ValidatingWebhookConfiguration", "name", validatingWebhookConfigName, "url", webhookURL) + if err := crClient.Create(ctx, webhookConfig); err != nil { + return fmt.Errorf("failed to create ValidatingWebhookConfiguration: %w", err) + } + + logger.V(1).Info("Created ValidatingWebhookConfiguration") + return nil +} + +func buildValidatingWebhook(webhookURL string, caCert []byte) admissionregistrationv1.ValidatingWebhook { + failurePolicy := admissionregistrationv1.Fail + sideEffects := admissionregistrationv1.SideEffectClassNone + matchPolicy := admissionregistrationv1.Exact + + return admissionregistrationv1.ValidatingWebhook{ + Name: "apiserviceexportrequests.kube-bind.io", + AdmissionReviewVersions: []string{"v1", "v1beta1"}, + ClientConfig: admissionregistrationv1.WebhookClientConfig{ + URL: &webhookURL, + CABundle: caCert, + }, + Rules: []admissionregistrationv1.RuleWithOperations{ + { + Operations: []admissionregistrationv1.OperationType{ + admissionregistrationv1.Create, + admissionregistrationv1.Update, + }, + Rule: admissionregistrationv1.Rule{ + APIGroups: []string{"kube-bind.io"}, + APIVersions: []string{"v1alpha2"}, + Resources: []string{"apiserviceexportrequests", "apiserviceexportrequest"}, + }, + }, + }, + FailurePolicy: &failurePolicy, + SideEffects: &sideEffects, + MatchPolicy: &matchPolicy, + NamespaceSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{}, + }, + } +} + +func GetWebhookURL(ctx context.Context, cfg *rest.Config, port int) (string, error) { + apiServerURL, err := url.Parse(cfg.Host) + if err != nil { + return "", fmt.Errorf("failed to parse API server URL: %w", err) + } + + hostname := apiServerURL.Hostname() + + if hostname == "127.0.0.1" || hostname == "::1" || hostname == "localhost" { + detectedHostname, err := detectHostnameFromCluster(ctx, cfg) + if err != nil { + hostname = "127.0.0.1" + } else { + hostname = detectedHostname + } + } + + return fmt.Sprintf( + "https://%s%s", + net.JoinHostPort(hostname, fmt.Sprintf("%d", port)), + WebhookPath, + ), nil +} + +func detectHostnameFromCluster(ctx context.Context, cfg *rest.Config) (string, error) { + logger := klog.FromContext(ctx) + + kubeClient, err := kubernetes.NewForConfig(cfg) + if err != nil { + return "", fmt.Errorf("failed to create kubernetes client: %w", err) + } + + nodes, err := kubeClient.CoreV1().Nodes().List(ctx, metav1.ListOptions{Limit: 1}) + if err != nil { + return "", fmt.Errorf("failed to list nodes: %w", err) + } + + if len(nodes.Items) == 0 { + return "", fmt.Errorf("no nodes found in cluster") + } + + node := nodes.Items[0] + + var nodeIP string + for _, addr := range node.Status.Addresses { + if addr.Type == corev1.NodeInternalIP { + nodeIP = addr.Address + break + } + } + + if nodeIP == "" { + return "", fmt.Errorf("node %s has no internal IP", node.Name) + } + + ip := net.ParseIP(nodeIP) + if ip == nil { + return "", fmt.Errorf("invalid node IP: %s", nodeIP) + } + + gatewayIP := inferGatewayFromNodeIP(ip) + if gatewayIP == nil { + return "", fmt.Errorf("could not infer gateway from node IP %s", nodeIP) + } + + gatewayStr := gatewayIP.String() + logger.V(1).Info("Inferred gateway IP from node IP", "nodeIP", nodeIP, "gatewayIP", gatewayStr) + return gatewayStr, nil +} + +func inferGatewayFromNodeIP(nodeIP net.IP) net.IP { + ipv4 := nodeIP.To4() + if ipv4 == nil { + return nil + } + + gateway := make(net.IP, len(ipv4)) + copy(gateway, ipv4) + + lastOctet := gateway[3] + if lastOctet < 255 { + gateway[3] = lastOctet + 1 + return gateway + } + + return nil +} + +func extractCABundle(certData []byte) ([]byte, error) { + var certs []*x509.Certificate + var block *pem.Block + rest := certData + + for { + block, rest = pem.Decode(rest) + if block == nil { + break + } + if block.Type == "CERTIFICATE" { + cert, err := x509.ParseCertificate(block.Bytes) + if err != nil { + continue + } + certs = append(certs, cert) + } + } + + if len(certs) == 0 { + return certData, nil + } + + if len(certs) > 0 && certs[0].IsCA { + return pem.EncodeToMemory(&pem.Block{ + Type: "CERTIFICATE", + Bytes: certs[0].Raw, + }), nil + } + + var caBundle []byte + for _, cert := range certs { + caBundle = append(caBundle, pem.EncodeToMemory(&pem.Block{ + Type: "CERTIFICATE", + Bytes: cert.Raw, + })...) + } + + if len(certs) == 1 { + return caBundle, nil + } + + return caBundle, nil +} diff --git a/test/e2e/framework/backend.go b/test/e2e/framework/backend.go index 843ebc356..074976c23 100644 --- a/test/e2e/framework/backend.go +++ b/test/e2e/framework/backend.go @@ -66,6 +66,12 @@ func StartBackend(t testing.TB, args ...string) (net.Addr, *backend.Server) { require.NoError(t, err) addr := opts.Serve.Listener.Addr() + webhookListener, err := net.Listen("tcp", "localhost:0") + require.NoError(t, err) + webhookAddr := webhookListener.Addr().(*net.TCPAddr) + opts.WebhookPort = webhookAddr.Port + webhookListener.Close() + opts.OIDC.Type = string(options.OIDCTypeEmbedded) opts.OIDC.IssuerURL = fmt.Sprintf("http://%s/oidc", addr.String()) opts.OIDC.CallbackURL = fmt.Sprintf("http://%s/api/callback", addr.String())