diff --git a/Makefile b/Makefile index 68a3264a..18469f2a 100644 --- a/Makefile +++ b/Makefile @@ -50,6 +50,7 @@ GO_LICENSES_VERSION=v2.0.1 GO_LICENSES_ENV=GOOS=linux GOARCH=amd64 GINKGO_VERSION = $(shell cat go.mod | grep 'github.com/onsi/ginkgo' | sed 's/.*\(v.*\)$$/\1/g') KIND_VERSION=v0.30.0 +CLOUD_PROVIDER_KIND_VERSION=v0.11.1 YQ_VERSION=v4.53.3 CONTROLLER_RUNTIME_VERSION = $(shell cat go.mod | grep 'sigs.k8s.io/controller-runtime' | sed 's/.*\(v\(.*\)\.[^.]*\)$$/\2/g') # ENVTEST_K8S_VERSION refers to the version of kubebuilder assets to be downloaded by envtest binary. @@ -327,7 +328,7 @@ e2e-tests: export OPERATOR_IMAGE=$(IMG):$(TAG) e2e-tests: export TEST_PARALLELISM?=4 # Use path for subcommands so that we use the correct dev-dependencies rather than those installed globally e2e-tests: export PATH:=$(LOCALBIN):${PATH} -e2e-tests: ginkgo kind manifests generate helm-dependency-build docker-build ## Run e2e/integration tests. For help, refer to: dev-docs/e2e-testing.md +e2e-tests: ginkgo kind cloud-provider-kind manifests generate helm-dependency-build docker-build ## Run e2e/integration tests. For help, refer to: dev-docs/e2e-testing.md ./tests/scripts/manage_e2e_tests.sh run-tests ##@ Helm @@ -387,7 +388,7 @@ LOCALBIN ?= $(PROJECT_DIR)/bin $(LOCALBIN): mkdir -p $(LOCALBIN) -install-dependencies: controller-gen kustomize go-licenses setup-envtest kind ginkgo ## Install necessary dependencies for building and testing the Solr Operator +install-dependencies: controller-gen kustomize go-licenses setup-envtest kind cloud-provider-kind ginkgo ## Install necessary dependencies for building and testing the Solr Operator CONTROLLER_GEN = $(LOCALBIN)/controller-gen .PHONY: controller-gen @@ -419,6 +420,12 @@ kind: $(KIND) ## Download kind locally if necessary. $(KIND): $(LOCALBIN) $(call go-get-tool,$(KIND),sigs.k8s.io/kind@$(KIND_VERSION)) +CLOUD_PROVIDER_KIND = $(LOCALBIN)/cloud-provider-kind +.PHONY: cloud-provider-kind +cloud-provider-kind: $(CLOUD_PROVIDER_KIND) ## Download cloud-provider-kin locally if necessary. +$(CLOUD_PROVIDER_KIND): $(LOCALBIN) + $(call go-get-tool,$(CLOUD_PROVIDER_KIND),sigs.k8s.io/cloud-provider-kind@$(CLOUD_PROVIDER_KIND_VERSION)) + YQ = $(LOCALBIN)/yq .PHONY: yq yq: $(YQ) ## Download yq locally if necessary. diff --git a/api/v1beta1/solrcloud_types.go b/api/v1beta1/solrcloud_types.go index 18930268..42076437 100644 --- a/api/v1beta1/solrcloud_types.go +++ b/api/v1beta1/solrcloud_types.go @@ -19,12 +19,13 @@ package v1beta1 import ( "fmt" - "github.com/go-logr/logr" - zkApi "github.com/pravega/zookeeper-operator/api/v1beta1" "math/rand" "strconv" "strings" + "github.com/go-logr/logr" + zkApi "github.com/pravega/zookeeper-operator/api/v1beta1" + "k8s.io/apimachinery/pkg/util/intstr" corev1 "k8s.io/api/core/v1" @@ -563,11 +564,20 @@ type ExternalAddressability struct { // // +optional IngressTLSTermination *SolrIngressTLSTermination `json:"ingressTLSTermination,omitempty"` + + // Gateway defines settings for Kubernetes Gateway API routing. + // + // This option is only available when Method=Gateway. + // The referenced Gateway must already exist and be managed by your platform team. + // The Solr Operator only manages the HTTPRoute resources. + // + // +optional + Gateway *SolrGatewayOptions `json:"gateway,omitempty"` } // ExternalAddressabilityMethod is a string enumeration type that enumerates // all possible ways that a SolrCloud can be made addressable external to the kubernetes cluster. -// +kubebuilder:validation:Enum=Ingress;ExternalDNS +// +kubebuilder:validation:Enum=Ingress;ExternalDNS;Gateway type ExternalAddressabilityMethod string const ( @@ -577,6 +587,9 @@ const ( // Use ExternalDNS to make the Solr service(s) externally addressable ExternalDNS ExternalAddressabilityMethod = "ExternalDNS" + // Use Gateway API to make the Solr service(s) externally addressable + Gateway ExternalAddressabilityMethod = "Gateway" + // Make Solr service(s) type:LoadBalancer to make them externally addressable // NOTE: This option is not currently supported. LoadBalancer ExternalAddressabilityMethod = "LoadBalancer" @@ -626,6 +639,107 @@ type SolrIngressTLSTermination struct { TLSSecret string `json:"tlsSecret,omitempty"` } +// SolrGatewayOptions defines how a SolrCloud should be exposed via Kubernetes Gateway API +type SolrGatewayOptions struct { + // ParentRefs specifies the Gateway(s) to attach HTTPRoutes to. + // This is required when using method=Gateway. + // + // The referenced Gateway must already exist and be managed by your platform team. + // The Solr Operator only manages the HTTPRoute resources. + // + // +kubebuilder:validation:MinItems=1 + ParentRefs []GatewayParentReference `json:"parentRefs"` + + // AdditionalHostnames specifies extra hostnames to include in the common HTTPRoute. + // These are appended to the auto-generated hostnames derived from DomainName and AdditionalDomainNames. + // This is useful for adding alias hostnames that should also route to the common Solr service. + // + // +optional + // +kubebuilder:validation:MaxItems=16 + AdditionalHostnames []string `json:"additionalHostnames,omitempty"` + + // Annotations to add to HTTPRoute resources + // +optional + Annotations map[string]string `json:"annotations,omitempty"` + + // Labels to add to HTTPRoute resources + // +optional + Labels map[string]string `json:"labels,omitempty"` + + // BackendTLSPolicy defines TLS configuration for backend connections from Gateway to Solr pods. + // + // This is used when Solr pods are running with TLS enabled (spec.solrTLS) and the Gateway + // needs to establish secure connections to the backend services. + // + // The Solr Operator will create BackendTLSPolicy resources for each HTTPRoute. + // + // +optional + BackendTLSPolicy *SolrBackendTLSPolicy `json:"backendTLSPolicy,omitempty"` +} + +// GatewayParentReference identifies a parent Gateway resource to attach HTTPRoutes to +type GatewayParentReference struct { + // Name of the Gateway resource + Name string `json:"name"` + + // Namespace of the Gateway resource. + // If not specified, defaults to the HTTPRoute's namespace. + // +optional + Namespace *string `json:"namespace,omitempty"` + + // SectionName refers to a specific listener on the Gateway. + // For example, "https" or "http". + // +optional + SectionName *string `json:"sectionName,omitempty"` +} + +// SolrBackendTLSPolicy defines backend TLS configuration for Gateway API +// +// For a valid BackendTLSPolicy configuration, exactly one of CACertificateRefs or +// WellKnownCACertificates must be specified. The operator validates this constraint +// via the HasBackendTLSPolicy() function before creating BackendTLSPolicy resources. +// +// +kubebuilder:validation:MaxProperties=1 +// +kubebuilder:validation:MinProperties=1 +type SolrBackendTLSPolicy struct { + // CACertificateRefs contains one or more references to Kubernetes objects that contain + // TLS certificates of the Certificate Authorities that can be used as a trust anchor + // to validate the certificates presented by the backend. + // + // If specified, WellKnownCACertificates must not be set. + // + // +optional + // +kubebuilder:validation:MaxItems=8 + CACertificateRefs []GatewayCertificateReference `json:"caCertificateRefs,omitempty"` + + // WellKnownCACertificates specifies whether system CA certificates may be used in the + // TLS handshake between the gateway and backend pod. + // + // If WellKnownCACertificates is unspecified or empty (""), then CACertificateRefs must be + // specified with at least one entry for a valid configuration. + // + // Only one of CACertificateRefs or WellKnownCACertificates may be specified, not both. + // If specified, CACertificateRefs must not be set. + // + // +optional + WellKnownCACertificates *string `json:"wellKnownCACertificates,omitempty"` +} + +// GatewayCertificateReference identifies a certificate object in Kubernetes +type GatewayCertificateReference struct { + // Name of the Kubernetes resource (e.g., ConfigMap or Secret) + Name string `json:"name"` + + // Kind of the resource (e.g., "ConfigMap" or "Secret") + // +optional + // +kubebuilder:default="ConfigMap" + Kind *string `json:"kind,omitempty"` + + // Group of the resource + // +optional + Group *string `json:"group,omitempty"` +} + type SolrUpdateStrategy struct { // Method defines the way in which SolrClouds should be updated when the podSpec changes. // +optional @@ -1297,6 +1411,26 @@ func (sc *SolrCloud) CommonIngressName() string { return fmt.Sprintf("%s-solrcloud-common", sc.GetName()) } +// CommonHTTPRouteName returns the name of the common HTTPRoute for the cloud +func (sc *SolrCloud) CommonHTTPRouteName() string { + return fmt.Sprintf("%s-solrcloud-common", sc.GetName()) +} + +// NodeHTTPRouteName returns the name of the HTTPRoute for a specific node +func (sc *SolrCloud) NodeHTTPRouteName(nodeName string) string { + return nodeName +} + +// CommonBackendTLSPolicyName returns the name of the common BackendTLSPolicy for the cloud +func (sc *SolrCloud) CommonBackendTLSPolicyName() string { + return fmt.Sprintf("%s-solrcloud-common", sc.GetName()) +} + +// NodeBackendTLSPolicyName returns the name of the BackendTLSPolicy for a specific node +func (sc *SolrCloud) NodeBackendTLSPolicyName(nodeName string) string { + return nodeName +} + // ProvidedZookeeperName returns the provided zk cluster func (sc *SolrCloud) ProvidedZookeeperName() string { return fmt.Sprintf("%s-solrcloud-zookeeper", sc.GetName()) @@ -1340,8 +1474,8 @@ func (sc *SolrCloud) UsesIndividualNodeServices() bool { } func (extOpts *ExternalAddressability) UsesIndividualNodeServices() bool { - // LoadBalancer and Ingress will not work with headless services if each pod needs to be exposed externally. - return extOpts != nil && !extOpts.HideNodes && (extOpts.Method == Ingress || extOpts.Method == LoadBalancer) + // LoadBalancer, Ingress, and Gateway will not work with headless services if each pod needs to be exposed externally. + return extOpts != nil && !extOpts.HideNodes && (extOpts.Method == Ingress || extOpts.Method == LoadBalancer || extOpts.Method == Gateway) } func (sc *SolrCloud) CommonExternalPrefix() string { @@ -1435,11 +1569,13 @@ func (sc *SolrCloud) ExternalNodeUrl(nodeName string, domainName string, withPor url = fmt.Sprintf("%s.%s", sc.NodeIngressPrefix(nodeName), domainName) } else if sc.Spec.SolrAddressability.External.Method == ExternalDNS { url = fmt.Sprintf("%s.%s", nodeName, sc.ExternalDnsDomain(domainName)) + } else if sc.Spec.SolrAddressability.External.Method == Gateway { + url = fmt.Sprintf("%s.%s", sc.NodeIngressPrefix(nodeName), domainName) } // TODO: Add LoadBalancer stuff here - if withPort && sc.Spec.SolrAddressability.External.Method != Ingress { - // Ingress does not require a port, since the port is whatever the ingress is listening on (80 and 443) + if withPort && sc.Spec.SolrAddressability.External.Method != Ingress && sc.Spec.SolrAddressability.External.Method != Gateway { + // Ingress and Gateway do not require a port, since the port is whatever the ingress/gateway is listening on (80 and 443) url += sc.NodePortSuffix(true) } return url @@ -1450,11 +1586,13 @@ func (sc *SolrCloud) ExternalCommonUrl(domainName string, withPort bool) (url st url = fmt.Sprintf("%s.%s", sc.CommonExternalPrefix(), domainName) } else if sc.Spec.SolrAddressability.External.Method == ExternalDNS { url = fmt.Sprintf("%s.%s", sc.CommonServiceName(), sc.ExternalDnsDomain(domainName)) + } else if sc.Spec.SolrAddressability.External.Method == Gateway { + url = fmt.Sprintf("%s.%s", sc.CommonExternalPrefix(), domainName) } // TODO: Add LoadBalancer stuff here - if withPort && sc.Spec.SolrAddressability.External.Method != Ingress { - // Ingress does not require a port, since the port is whatever the ingress is listening on (80 and 443) + if withPort && sc.Spec.SolrAddressability.External.Method != Ingress && sc.Spec.SolrAddressability.External.Method != Gateway { + // Ingress and Gateway do not require a port, since the port is whatever the ingress/gateway is listening on (80 and 443) url += sc.CommonPortSuffix(true) } return url @@ -1467,6 +1605,14 @@ func (ea *ExternalAddressability) HasIngressTLSTermination() bool { return false } +func (ea *ExternalAddressability) HasBackendTLSPolicy() bool { + if ea != nil && ea.Method == Gateway && ea.Gateway != nil && ea.Gateway.BackendTLSPolicy != nil { + return (ea.Gateway.BackendTLSPolicy.CACertificateRefs != nil && len(ea.Gateway.BackendTLSPolicy.CACertificateRefs) > 0) || + (ea.Gateway.BackendTLSPolicy.WellKnownCACertificates != nil && *ea.Gateway.BackendTLSPolicy.WellKnownCACertificates != "") + } + return false +} + func (sc *SolrCloud) UrlScheme(external bool) string { urlScheme := "http" if sc.Spec.SolrTLS != nil { diff --git a/api/v1beta1/zz_generated.deepcopy.go b/api/v1beta1/zz_generated.deepcopy.go index b3e4dc64..ff45ba52 100644 --- a/api/v1beta1/zz_generated.deepcopy.go +++ b/api/v1beta1/zz_generated.deepcopy.go @@ -263,6 +263,11 @@ func (in *ExternalAddressability) DeepCopyInto(out *ExternalAddressability) { *out = new(SolrIngressTLSTermination) **out = **in } + if in.Gateway != nil { + in, out := &in.Gateway, &out.Gateway + *out = new(SolrGatewayOptions) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ExternalAddressability. @@ -275,6 +280,56 @@ func (in *ExternalAddressability) DeepCopy() *ExternalAddressability { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *GatewayCertificateReference) DeepCopyInto(out *GatewayCertificateReference) { + *out = *in + if in.Kind != nil { + in, out := &in.Kind, &out.Kind + *out = new(string) + **out = **in + } + if in.Group != nil { + in, out := &in.Group, &out.Group + *out = new(string) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GatewayCertificateReference. +func (in *GatewayCertificateReference) DeepCopy() *GatewayCertificateReference { + if in == nil { + return nil + } + out := new(GatewayCertificateReference) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *GatewayParentReference) DeepCopyInto(out *GatewayParentReference) { + *out = *in + if in.Namespace != nil { + in, out := &in.Namespace, &out.Namespace + *out = new(string) + **out = **in + } + if in.SectionName != nil { + in, out := &in.SectionName, &out.SectionName + *out = new(string) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GatewayParentReference. +func (in *GatewayParentReference) DeepCopy() *GatewayParentReference { + if in == nil { + return nil + } + out := new(GatewayParentReference) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *GcsRepository) DeepCopyInto(out *GcsRepository) { *out = *in @@ -678,6 +733,33 @@ func (in *SolrAvailabilityOptions) DeepCopy() *SolrAvailabilityOptions { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SolrBackendTLSPolicy) DeepCopyInto(out *SolrBackendTLSPolicy) { + *out = *in + if in.CACertificateRefs != nil { + in, out := &in.CACertificateRefs, &out.CACertificateRefs + *out = make([]GatewayCertificateReference, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.WellKnownCACertificates != nil { + in, out := &in.WellKnownCACertificates, &out.WellKnownCACertificates + *out = new(string) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SolrBackendTLSPolicy. +func (in *SolrBackendTLSPolicy) DeepCopy() *SolrBackendTLSPolicy { + if in == nil { + return nil + } + out := new(SolrBackendTLSPolicy) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *SolrBackup) DeepCopyInto(out *SolrBackup) { *out = *in @@ -1054,6 +1136,52 @@ func (in *SolrEphemeralDataStorageOptions) DeepCopy() *SolrEphemeralDataStorageO return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SolrGatewayOptions) DeepCopyInto(out *SolrGatewayOptions) { + *out = *in + if in.ParentRefs != nil { + in, out := &in.ParentRefs, &out.ParentRefs + *out = make([]GatewayParentReference, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.AdditionalHostnames != nil { + in, out := &in.AdditionalHostnames, &out.AdditionalHostnames + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.Annotations != nil { + in, out := &in.Annotations, &out.Annotations + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.Labels != nil { + in, out := &in.Labels, &out.Labels + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.BackendTLSPolicy != nil { + in, out := &in.BackendTLSPolicy, &out.BackendTLSPolicy + *out = new(SolrBackendTLSPolicy) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SolrGatewayOptions. +func (in *SolrGatewayOptions) DeepCopy() *SolrGatewayOptions { + if in == nil { + return nil + } + out := new(SolrGatewayOptions) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *SolrIngressTLSTermination) DeepCopyInto(out *SolrIngressTLSTermination) { *out = *in diff --git a/config/crd/bases/solr.apache.org_solrclouds.yaml b/config/crd/bases/solr.apache.org_solrclouds.yaml index 7aab16f8..5475738f 100644 --- a/config/crd/bases/solr.apache.org_solrclouds.yaml +++ b/config/crd/bases/solr.apache.org_solrclouds.yaml @@ -10709,6 +10709,115 @@ spec: For the LoadBalancer method, this field is optional and will only be used when useExternalAddress=true. If used with the LoadBalancer method, you will need DNS routing to the LoadBalancer IP address through the url template given above. type: string + gateway: + description: |- + Gateway defines settings for Kubernetes Gateway API routing. + + This option is only available when Method=Gateway. + The referenced Gateway must already exist and be managed by your platform team. + The Solr Operator only manages the HTTPRoute resources. + properties: + additionalHostnames: + description: |- + AdditionalHostnames specifies extra hostnames to include in the common HTTPRoute. + These are appended to the auto-generated hostnames derived from DomainName and AdditionalDomainNames. + This is useful for adding alias hostnames that should also route to the common Solr service. + items: + type: string + maxItems: 16 + type: array + annotations: + additionalProperties: + type: string + description: Annotations to add to HTTPRoute resources + type: object + backendTLSPolicy: + description: |- + BackendTLSPolicy defines TLS configuration for backend connections from Gateway to Solr pods. + + This is used when Solr pods are running with TLS enabled (spec.solrTLS) and the Gateway + needs to establish secure connections to the backend services. + + The Solr Operator will create BackendTLSPolicy resources for each HTTPRoute. + maxProperties: 1 + properties: + caCertificateRefs: + description: |- + CACertificateRefs contains one or more references to Kubernetes objects that contain + TLS certificates of the Certificate Authorities that can be used as a trust anchor + to validate the certificates presented by the backend. + + If specified, WellKnownCACertificates must not be set. + items: + description: GatewayCertificateReference identifies + a certificate object in Kubernetes + properties: + group: + description: Group of the resource + type: string + kind: + default: ConfigMap + description: Kind of the resource (e.g., "ConfigMap" + or "Secret") + type: string + name: + description: Name of the Kubernetes resource + (e.g., ConfigMap or Secret) + type: string + required: + - name + type: object + maxItems: 8 + type: array + wellKnownCACertificates: + description: |- + WellKnownCACertificates specifies whether system CA certificates may be used in the + TLS handshake between the gateway and backend pod. + + If WellKnownCACertificates is unspecified or empty (""), then CACertificateRefs must be + specified with at least one entry for a valid configuration. + + Only one of CACertificateRefs or WellKnownCACertificates may be specified, not both. + If specified, CACertificateRefs must not be set. + type: string + type: object + labels: + additionalProperties: + type: string + description: Labels to add to HTTPRoute resources + type: object + parentRefs: + description: |- + ParentRefs specifies the Gateway(s) to attach HTTPRoutes to. + This is required when using method=Gateway. + + The referenced Gateway must already exist and be managed by your platform team. + The Solr Operator only manages the HTTPRoute resources. + items: + description: GatewayParentReference identifies a parent + Gateway resource to attach HTTPRoutes to + properties: + name: + description: Name of the Gateway resource + type: string + namespace: + description: |- + Namespace of the Gateway resource. + If not specified, defaults to the HTTPRoute's namespace. + type: string + sectionName: + description: |- + SectionName refers to a specific listener on the Gateway. + For example, "https" or "http". + type: string + required: + - name + type: object + minItems: 1 + type: array + required: + - parentRefs + type: object hideCommon: description: |- Do not expose the common Solr service externally. This affects a single service. @@ -10748,6 +10857,7 @@ spec: enum: - Ingress - ExternalDNS + - Gateway type: string nodePortOverride: description: |- diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index a6b37dbe..884d479c 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -99,6 +99,26 @@ rules: - statefulsets/status verbs: - get +- apiGroups: + - gateway.networking.k8s.io + resources: + - backendtlspolicies + - httproutes + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - gateway.networking.k8s.io + resources: + - backendtlspolicies/status + - httproutes/status + verbs: + - get - apiGroups: - networking.k8s.io resources: diff --git a/controllers/solr_gateway_util.go b/controllers/solr_gateway_util.go new file mode 100644 index 00000000..0fe25125 --- /dev/null +++ b/controllers/solr_gateway_util.go @@ -0,0 +1,339 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package controllers + +import ( + "context" + "fmt" + + solrv1beta1 "github.com/apache/solr-operator/api/v1beta1" + "github.com/apache/solr-operator/controllers/util" + "github.com/go-logr/logr" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + gatewayv1 "sigs.k8s.io/gateway-api/apis/v1" +) + +func (r *SolrCloudReconciler) reconcileGatewayResources(ctx context.Context, cloud *solrv1beta1.SolrCloud, solrNodeNames []string, logger logr.Logger) error { + extAddressabilityOpts := cloud.Spec.SolrAddressability.External + + // If not using Gateway method, clean up any existing HTTPRoutes and BackendTLSPolicy resources + if extAddressabilityOpts == nil || extAddressabilityOpts.Method != solrv1beta1.Gateway { + return r.cleanupGatewayResources(ctx, cloud, logger) + } + + // Validate that gateway config is provided + if extAddressabilityOpts.Gateway == nil || len(extAddressabilityOpts.Gateway.ParentRefs) == 0 { + r.Recorder.Event(cloud, corev1.EventTypeWarning, "InvalidConfiguration", "gateway.parentRefs is required when using method=Gateway") + return fmt.Errorf("gateway.parentRefs is required when using method=Gateway") + } + + // Validate that BackendTLSPolicy requires solrTLS to be configured + if extAddressabilityOpts.HasBackendTLSPolicy() && cloud.Spec.SolrTLS == nil { + r.Recorder.Event(cloud, corev1.EventTypeWarning, "InvalidConfiguration", "gateway.backendTLSPolicy requires spec.solrTLS to be configured, because BackendTLSPolicy instructs the gateway to use TLS when connecting to Solr backends") + return fmt.Errorf("invalid config: `spec.customSolrKubeOptions.addressability.external.gateway.backendTLSPolicy` requires `spec.solrTLS` to be configured, because BackendTLSPolicy instructs the gateway to use TLS when connecting to Solr backends") + } + + if err := r.reconcileGatewayHTTPRoutes(ctx, cloud, extAddressabilityOpts, solrNodeNames, logger); err != nil { + return err + } + + return r.reconcileGatewayBackendTLSPolicies(ctx, cloud, extAddressabilityOpts, solrNodeNames, logger) +} + +func (r *SolrCloudReconciler) reconcileGatewayHTTPRoutes(ctx context.Context, cloud *solrv1beta1.SolrCloud, extAddressabilityOpts *solrv1beta1.ExternalAddressability, solrNodeNames []string, logger logr.Logger) error { + // Reconcile Common HTTPRoute (if not hidden) + if !extAddressabilityOpts.HideCommon { + commonHTTPRoute := util.GenerateCommonHTTPRoute(cloud, solrNodeNames) + commonHTTPRouteLogger := logger.WithValues("httproute", commonHTTPRoute.Name) + foundCommonHTTPRoute := &gatewayv1.HTTPRoute{} + err := r.Get(ctx, types.NamespacedName{Name: commonHTTPRoute.Name, Namespace: commonHTTPRoute.Namespace}, foundCommonHTTPRoute) + if err != nil && errors.IsNotFound(err) { + commonHTTPRouteLogger.Info("Creating common HTTPRoute") + if err = controllerutil.SetControllerReference(cloud, commonHTTPRoute, r.Scheme); err == nil { + err = r.Create(ctx, commonHTTPRoute) + } + } else if err == nil { + var needsUpdate bool + needsUpdate, err = util.OvertakeControllerRef(cloud, foundCommonHTTPRoute, r.Scheme) + needsUpdate = util.CopyHTTPRouteFields(commonHTTPRoute, foundCommonHTTPRoute, commonHTTPRouteLogger) || needsUpdate + + if needsUpdate && err == nil { + commonHTTPRouteLogger.Info("Updating common HTTPRoute") + err = r.Update(ctx, foundCommonHTTPRoute) + } + } + if err != nil { + return err + } + } else { + // Delete common HTTPRoute if it exists but should be hidden + foundCommonHTTPRoute := &gatewayv1.HTTPRoute{} + err := r.Get(ctx, types.NamespacedName{Name: cloud.CommonHTTPRouteName(), Namespace: cloud.GetNamespace()}, foundCommonHTTPRoute) + if err == nil { + logger.Info("Deleting common HTTPRoute (hideCommon=true)") + err = r.Delete(ctx, foundCommonHTTPRoute) + if err != nil && !errors.IsNotFound(err) { + return err + } + } + } + + // Reconcile Node HTTPRoutes (if not hidden) + if !extAddressabilityOpts.HideNodes { + for _, nodeName := range solrNodeNames { + nodeHTTPRoute := util.GenerateNodeHTTPRoute(cloud, nodeName) + nodeHTTPRouteLogger := logger.WithValues("httproute", nodeHTTPRoute.Name) + foundNodeHTTPRoute := &gatewayv1.HTTPRoute{} + err := r.Get(ctx, types.NamespacedName{Name: nodeHTTPRoute.Name, Namespace: nodeHTTPRoute.Namespace}, foundNodeHTTPRoute) + if err != nil && errors.IsNotFound(err) { + nodeHTTPRouteLogger.Info("Creating node HTTPRoute") + if err = controllerutil.SetControllerReference(cloud, nodeHTTPRoute, r.Scheme); err == nil { + err = r.Create(ctx, nodeHTTPRoute) + } + } else if err == nil { + var needsUpdate bool + needsUpdate, err = util.OvertakeControllerRef(cloud, foundNodeHTTPRoute, r.Scheme) + needsUpdate = util.CopyHTTPRouteFields(nodeHTTPRoute, foundNodeHTTPRoute, nodeHTTPRouteLogger) || needsUpdate + + if needsUpdate && err == nil { + nodeHTTPRouteLogger.Info("Updating node HTTPRoute") + err = r.Update(ctx, foundNodeHTTPRoute) + } + } + if err != nil { + return err + } + } + } + + // Cleanup orphaned node HTTPRoutes (when scaling down or hideNodes=true) + httpRouteList := &gatewayv1.HTTPRouteList{} + listOps := &client.ListOptions{ + Namespace: cloud.Namespace, + LabelSelector: labels.SelectorFromSet(cloud.SharedLabels()), + } + if err := r.List(ctx, httpRouteList, listOps); err != nil { + return err + } + for _, httpRoute := range httpRouteList.Items { + // Skip the common HTTPRoute + if httpRoute.Name == cloud.CommonHTTPRouteName() { + continue + } + + // Delete if hideNodes is true, or if this node no longer exists (scale-down) + shouldDelete := extAddressabilityOpts.HideNodes + if !shouldDelete { + nodeExists := false + for _, nodeName := range solrNodeNames { + if httpRoute.Name == cloud.NodeHTTPRouteName(nodeName) { + nodeExists = true + break + } + } + shouldDelete = !nodeExists + } + + if shouldDelete { + if extAddressabilityOpts.HideNodes { + logger.Info("Deleting node HTTPRoute (hideNodes=true)", "httproute", httpRoute.Name) + } else { + logger.Info("Deleting orphaned node HTTPRoute", "httproute", httpRoute.Name) + } + if err := r.Delete(ctx, &httpRoute); err != nil && !errors.IsNotFound(err) { + return err + } + } + } + + return nil +} + +func (r *SolrCloudReconciler) reconcileGatewayBackendTLSPolicies(ctx context.Context, cloud *solrv1beta1.SolrCloud, extAddressabilityOpts *solrv1beta1.ExternalAddressability, solrNodeNames []string, logger logr.Logger) error { + listOps := &client.ListOptions{ + Namespace: cloud.Namespace, + LabelSelector: labels.SelectorFromSet(cloud.SharedLabels()), + } + + // If BackendTLSPolicy is not configured, clean up any existing BackendTLSPolicy resources + if !extAddressabilityOpts.HasBackendTLSPolicy() { + backendTLSPolicyList := &gatewayv1.BackendTLSPolicyList{} + if err := r.List(ctx, backendTLSPolicyList, listOps); err != nil { + return err + } + if len(backendTLSPolicyList.Items) > 0 { + logger.Info("Cleaning up BackendTLSPolicy resources (BackendTLSPolicy disabled)") + for _, policy := range backendTLSPolicyList.Items { + if err := r.Delete(ctx, &policy); err != nil && !errors.IsNotFound(err) { + return err + } + } + } + return nil + } + + // Reconcile Common BackendTLSPolicy (if not hidden) + if !extAddressabilityOpts.HideCommon { + commonPolicy := util.GenerateCommonBackendTLSPolicy(cloud) + if commonPolicy != nil { + commonPolicyLogger := logger.WithValues("backendtlspolicy", commonPolicy.Name) + foundCommonPolicy := &gatewayv1.BackendTLSPolicy{} + err := r.Get(ctx, types.NamespacedName{Name: commonPolicy.Name, Namespace: commonPolicy.Namespace}, foundCommonPolicy) + if err != nil && errors.IsNotFound(err) { + commonPolicyLogger.Info("Creating common BackendTLSPolicy") + if err = controllerutil.SetControllerReference(cloud, commonPolicy, r.Scheme); err == nil { + err = r.Create(ctx, commonPolicy) + } + } else if err == nil { + var needsUpdate bool + needsUpdate, err = util.OvertakeControllerRef(cloud, foundCommonPolicy, r.Scheme) + needsUpdate = util.CopyBackendTLSPolicyFields(commonPolicy, foundCommonPolicy, commonPolicyLogger) || needsUpdate + + if needsUpdate && err == nil { + commonPolicyLogger.Info("Updating common BackendTLSPolicy") + err = r.Update(ctx, foundCommonPolicy) + } + } + if err != nil { + return err + } + } + } else { + // Delete common BackendTLSPolicy if it exists but should be hidden + foundCommonPolicy := &gatewayv1.BackendTLSPolicy{} + err := r.Get(ctx, types.NamespacedName{Name: cloud.CommonBackendTLSPolicyName(), Namespace: cloud.GetNamespace()}, foundCommonPolicy) + if err == nil { + logger.Info("Deleting common BackendTLSPolicy (hideCommon=true)") + err = r.Delete(ctx, foundCommonPolicy) + if err != nil && !errors.IsNotFound(err) { + return err + } + } + } + + // Reconcile Node BackendTLSPolicies (if not hidden) + if !extAddressabilityOpts.HideNodes { + for _, nodeName := range solrNodeNames { + nodePolicy := util.GenerateNodeBackendTLSPolicy(cloud, nodeName) + if nodePolicy != nil { + nodePolicyLogger := logger.WithValues("backendtlspolicy", nodePolicy.Name) + foundNodePolicy := &gatewayv1.BackendTLSPolicy{} + err := r.Get(ctx, types.NamespacedName{Name: nodePolicy.Name, Namespace: nodePolicy.Namespace}, foundNodePolicy) + if err != nil && errors.IsNotFound(err) { + nodePolicyLogger.Info("Creating node BackendTLSPolicy") + if err = controllerutil.SetControllerReference(cloud, nodePolicy, r.Scheme); err == nil { + err = r.Create(ctx, nodePolicy) + } + } else if err == nil { + var needsUpdate bool + needsUpdate, err = util.OvertakeControllerRef(cloud, foundNodePolicy, r.Scheme) + needsUpdate = util.CopyBackendTLSPolicyFields(nodePolicy, foundNodePolicy, nodePolicyLogger) || needsUpdate + + if needsUpdate && err == nil { + nodePolicyLogger.Info("Updating node BackendTLSPolicy") + err = r.Update(ctx, foundNodePolicy) + } + } + if err != nil { + return err + } + } + } + } + + // Cleanup node BackendTLSPolicies (when hideNodes is true or when scaling down) + backendTLSPolicyList := &gatewayv1.BackendTLSPolicyList{} + if err := r.List(ctx, backendTLSPolicyList, listOps); err != nil { + return err + } + for _, policy := range backendTLSPolicyList.Items { + // Skip the common BackendTLSPolicy + if policy.Name == cloud.CommonBackendTLSPolicyName() { + continue + } + + // Delete if hideNodes is true, or if this node no longer exists (scale-down) + shouldDelete := extAddressabilityOpts.HideNodes + if !shouldDelete { + // Check if this node still exists + nodeExists := false + for _, nodeName := range solrNodeNames { + if policy.Name == cloud.NodeBackendTLSPolicyName(nodeName) { + nodeExists = true + break + } + } + shouldDelete = !nodeExists + } + + if shouldDelete { + if extAddressabilityOpts.HideNodes { + logger.Info("Deleting node BackendTLSPolicy (hideNodes=true)", "backendtlspolicy", policy.Name) + } else { + logger.Info("Deleting orphaned node BackendTLSPolicy", "backendtlspolicy", policy.Name) + } + if err := r.Delete(ctx, &policy); err != nil && !errors.IsNotFound(err) { + return err + } + } + } + + return nil +} + +func (r *SolrCloudReconciler) cleanupGatewayResources(ctx context.Context, cloud *solrv1beta1.SolrCloud, logger logr.Logger) error { + listOps := &client.ListOptions{ + Namespace: cloud.Namespace, + LabelSelector: labels.SelectorFromSet(cloud.SharedLabels()), + } + + // Clean up HTTPRoutes + httpRouteList := &gatewayv1.HTTPRouteList{} + if err := r.List(ctx, httpRouteList, listOps); err != nil { + return err + } + if len(httpRouteList.Items) > 0 { + logger.Info("Cleaning up HTTPRoutes (method changed from Gateway)") + for _, httpRoute := range httpRouteList.Items { + if err := r.Delete(ctx, &httpRoute); err != nil && !errors.IsNotFound(err) { + return err + } + } + } + + // Clean up BackendTLSPolicy resources + backendTLSPolicyList := &gatewayv1.BackendTLSPolicyList{} + if err := r.List(ctx, backendTLSPolicyList, listOps); err != nil { + return err + } + if len(backendTLSPolicyList.Items) > 0 { + logger.Info("Cleaning up BackendTLSPolicy resources (method changed from Gateway)") + for _, policy := range backendTLSPolicyList.Items { + if err := r.Delete(ctx, &policy); err != nil && !errors.IsNotFound(err) { + return err + } + } + } + + return nil +} diff --git a/controllers/solrcloud_controller.go b/controllers/solrcloud_controller.go index cc0b2496..efe222e8 100644 --- a/controllers/solrcloud_controller.go +++ b/controllers/solrcloud_controller.go @@ -52,6 +52,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/predicate" "sigs.k8s.io/controller-runtime/pkg/reconcile" + gatewayv1 "sigs.k8s.io/gateway-api/apis/v1" ) // SolrCloudReconciler reconciles a SolrCloud object @@ -63,10 +64,18 @@ type SolrCloudReconciler struct { var useZkCRD bool +// useGatewayAPI is set when the Gateway API CRDs (both HTTPRoute and BackendTLSPolicy) +// are installed in the cluster. This is detected at startup rather than configured via a flag. +var useGatewayAPI bool + func UseZkCRD(useCRD bool) { useZkCRD = useCRD } +func UseGatewayAPI(useGateway bool) { + useGatewayAPI = useGateway +} + //+kubebuilder:rbac:groups="",resources=pods,verbs=get;list;watch;delete //+kubebuilder:rbac:groups="",resources=pods/status,verbs=get;patch //+kubebuilder:rbac:groups="",resources=events,verbs=create;patch @@ -76,6 +85,10 @@ func UseZkCRD(useCRD bool) { //+kubebuilder:rbac:groups=apps,resources=statefulsets/status,verbs=get //+kubebuilder:rbac:groups=networking.k8s.io,resources=ingresses,verbs=get;list;watch;create;update;patch;delete //+kubebuilder:rbac:groups=networking.k8s.io,resources=ingresses/status,verbs=get +//+kubebuilder:rbac:groups=gateway.networking.k8s.io,resources=httproutes,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups=gateway.networking.k8s.io,resources=httproutes/status,verbs=get +//+kubebuilder:rbac:groups=gateway.networking.k8s.io,resources=backendtlspolicies,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups=gateway.networking.k8s.io,resources=backendtlspolicies/status,verbs=get //+kubebuilder:rbac:groups="",resources=configmaps,verbs=get;list;watch;create;update;patch;delete //+kubebuilder:rbac:groups="",resources=configmaps/status,verbs=get //+kubebuilder:rbac:groups="",resources=persistentvolumeclaims,verbs=get;list;watch;update;patch;delete @@ -450,6 +463,13 @@ func (r *SolrCloudReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( } } + // Reconcile Gateway API HTTPRoutes + if useGatewayAPI { + if err = r.reconcileGatewayResources(ctx, instance, solrNodeNames, logger); err != nil { + return requeueOrNot, err + } + } + // ********************************************************* // The operations after this require a statefulSet to exist, // including updating the solrCloud status @@ -1391,6 +1411,10 @@ func (r *SolrCloudReconciler) SetupWithManager(mgr ctrl.Manager) error { ctrlBuilder = ctrlBuilder.Owns(&zkApi.ZookeeperCluster{}) } + if useGatewayAPI { + ctrlBuilder = ctrlBuilder.Owns(&gatewayv1.HTTPRoute{}).Owns(&gatewayv1.BackendTLSPolicy{}) + } + return ctrlBuilder.Complete(r) } diff --git a/controllers/util/gateway_util.go b/controllers/util/gateway_util.go new file mode 100644 index 00000000..98b83182 --- /dev/null +++ b/controllers/util/gateway_util.go @@ -0,0 +1,214 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package util + +import ( + solr "github.com/apache/solr-operator/api/v1beta1" + "github.com/go-logr/logr" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + gatewayv1 "sigs.k8s.io/gateway-api/apis/v1" +) + +// GenerateCommonHTTPRoute creates an HTTPRoute for the common Solr service +func GenerateCommonHTTPRoute(solrCloud *solr.SolrCloud, nodeNames []string) *gatewayv1.HTTPRoute { + labels := solrCloud.SharedLabelsWith(solrCloud.GetLabels()) + var annotations map[string]string + + gatewayOpts := solrCloud.Spec.SolrAddressability.External.Gateway + if gatewayOpts != nil { + labels = MergeLabelsOrAnnotations(labels, gatewayOpts.Labels) + annotations = MergeLabelsOrAnnotations(annotations, gatewayOpts.Annotations) + } + + extOpts := solrCloud.Spec.SolrAddressability.External + + // Create advertised domain name and possible additional domain names + allDomains := append([]string{extOpts.DomainName}, extOpts.AdditionalDomainNames...) + hostnames := make([]gatewayv1.Hostname, 0, len(allDomains)+len(gatewayOpts.AdditionalHostnames)) + for _, domain := range allDomains { + hostname := gatewayv1.Hostname(solrCloud.ExternalCommonUrl(domain, false)) + hostnames = append(hostnames, hostname) + } + // Append user-specified additional hostnames for the common route + for _, h := range gatewayOpts.AdditionalHostnames { + hostnames = append(hostnames, gatewayv1.Hostname(h)) + } + + // Convert parentRefs from our type to Gateway API type + parentRefs := make([]gatewayv1.ParentReference, len(gatewayOpts.ParentRefs)) + for i, ref := range gatewayOpts.ParentRefs { + parentRef := gatewayv1.ParentReference{ + Name: gatewayv1.ObjectName(ref.Name), + } + if ref.Namespace != nil { + namespace := gatewayv1.Namespace(*ref.Namespace) + parentRef.Namespace = &namespace + } + if ref.SectionName != nil { + sectionName := gatewayv1.SectionName(*ref.SectionName) + parentRef.SectionName = §ionName + } + parentRefs[i] = parentRef + } + + // Determine backend port + backendPort := gatewayv1.PortNumber(solrCloud.Spec.SolrAddressability.CommonServicePort) + + httpRoute := &gatewayv1.HTTPRoute{ + ObjectMeta: metav1.ObjectMeta{ + Name: solrCloud.CommonHTTPRouteName(), + Namespace: solrCloud.GetNamespace(), + Labels: labels, + Annotations: annotations, + }, + Spec: gatewayv1.HTTPRouteSpec{ + CommonRouteSpec: gatewayv1.CommonRouteSpec{ + ParentRefs: parentRefs, + }, + Hostnames: hostnames, + Rules: []gatewayv1.HTTPRouteRule{ + { + BackendRefs: []gatewayv1.HTTPBackendRef{ + { + BackendRef: gatewayv1.BackendRef{ + BackendObjectReference: gatewayv1.BackendObjectReference{ + Name: gatewayv1.ObjectName(solrCloud.CommonServiceName()), + Port: &backendPort, + }, + }, + }, + }, + }, + }, + }, + } + + return httpRoute +} + +// GenerateNodeHTTPRoute creates an HTTPRoute for individual Solr nodes +func GenerateNodeHTTPRoute(solrCloud *solr.SolrCloud, nodeName string) *gatewayv1.HTTPRoute { + labels := solrCloud.SharedLabelsWith(solrCloud.GetLabels()) + var annotations map[string]string + + gatewayOpts := solrCloud.Spec.SolrAddressability.External.Gateway + labels = MergeLabelsOrAnnotations(labels, gatewayOpts.Labels) + annotations = MergeLabelsOrAnnotations(annotations, gatewayOpts.Annotations) + + extOpts := solrCloud.Spec.SolrAddressability.External + + // Create hostnames for all domains + allDomains := append([]string{extOpts.DomainName}, extOpts.AdditionalDomainNames...) + hostnames := make([]gatewayv1.Hostname, 0, len(allDomains)) + for _, domain := range allDomains { + hostname := gatewayv1.Hostname(solrCloud.ExternalNodeUrl(nodeName, domain, false)) + hostnames = append(hostnames, hostname) + } + + // Convert parentRefs + parentRefs := make([]gatewayv1.ParentReference, len(gatewayOpts.ParentRefs)) + for i, ref := range gatewayOpts.ParentRefs { + parentRef := gatewayv1.ParentReference{ + Name: gatewayv1.ObjectName(ref.Name), + } + if ref.Namespace != nil { + namespace := gatewayv1.Namespace(*ref.Namespace) + parentRef.Namespace = &namespace + } + if ref.SectionName != nil { + sectionName := gatewayv1.SectionName(*ref.SectionName) + parentRef.SectionName = §ionName + } + parentRefs[i] = parentRef + } + + // Determine backend port (uses NodePort which may be overridden) + backendPort := gatewayv1.PortNumber(solrCloud.NodePort()) + + httpRoute := &gatewayv1.HTTPRoute{ + ObjectMeta: metav1.ObjectMeta{ + Name: solrCloud.NodeHTTPRouteName(nodeName), + Namespace: solrCloud.GetNamespace(), + Labels: labels, + Annotations: annotations, + }, + Spec: gatewayv1.HTTPRouteSpec{ + CommonRouteSpec: gatewayv1.CommonRouteSpec{ + ParentRefs: parentRefs, + }, + Hostnames: hostnames, + Rules: []gatewayv1.HTTPRouteRule{ + { + BackendRefs: []gatewayv1.HTTPBackendRef{ + { + BackendRef: gatewayv1.BackendRef{ + BackendObjectReference: gatewayv1.BackendObjectReference{ + Name: gatewayv1.ObjectName(nodeName), + Port: &backendPort, + }, + }, + }, + }, + }, + }, + }, + } + + return httpRoute +} + +// CopyHTTPRouteFields copies the fields from one HTTPRoute to another +// Returns true if there are differences between the two objects +func CopyHTTPRouteFields(from, to *gatewayv1.HTTPRoute, logger logr.Logger) bool { + requireUpdate := false + + // Copy labels + if !DeepEqualWithNils(to.Labels, from.Labels) { + logger.Info("HTTPRoute labels have changed") + requireUpdate = true + to.Labels = from.Labels + } + + // Copy annotations + if !DeepEqualWithNils(to.Annotations, from.Annotations) { + logger.Info("HTTPRoute annotations have changed") + requireUpdate = true + to.Annotations = from.Annotations + } + + // Copy spec + if !DeepEqualWithNils(to.Spec.ParentRefs, from.Spec.ParentRefs) { + logger.Info("HTTPRoute parentRefs have changed") + requireUpdate = true + to.Spec.ParentRefs = from.Spec.ParentRefs + } + + if !DeepEqualWithNils(to.Spec.Hostnames, from.Spec.Hostnames) { + logger.Info("HTTPRoute hostnames have changed") + requireUpdate = true + to.Spec.Hostnames = from.Spec.Hostnames + } + + if !DeepEqualWithNils(to.Spec.Rules, from.Spec.Rules) { + logger.Info("HTTPRoute rules have changed") + requireUpdate = true + to.Spec.Rules = from.Spec.Rules + } + + return requireUpdate +} diff --git a/controllers/util/gateway_util_backendtls.go b/controllers/util/gateway_util_backendtls.go new file mode 100644 index 00000000..3be49513 --- /dev/null +++ b/controllers/util/gateway_util_backendtls.go @@ -0,0 +1,192 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package util + +import ( + solr "github.com/apache/solr-operator/api/v1beta1" + "github.com/go-logr/logr" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + gatewayv1 "sigs.k8s.io/gateway-api/apis/v1" +) + +// GenerateCommonBackendTLSPolicy creates a BackendTLSPolicy for the common Solr service +func GenerateCommonBackendTLSPolicy(solrCloud *solr.SolrCloud) *gatewayv1.BackendTLSPolicy { + if solrCloud.Spec.SolrAddressability.External.Gateway == nil || + solrCloud.Spec.SolrAddressability.External.Gateway.BackendTLSPolicy == nil { + return nil + } + + // Get the full FQDN for hostname validation + domainName := solrCloud.Spec.SolrAddressability.External.DomainName + fqdn := solrCloud.ExternalCommonUrl(domainName, false) + + labels := solrCloud.SharedLabelsWith(solrCloud.GetLabels()) + backendTLSConfig := solrCloud.Spec.SolrAddressability.External.Gateway.BackendTLSPolicy + + // Convert CA certificate refs + var caCertRefs []gatewayv1.LocalObjectReference + if backendTLSConfig.CACertificateRefs != nil { + caCertRefs = make([]gatewayv1.LocalObjectReference, len(backendTLSConfig.CACertificateRefs)) + for i, ref := range backendTLSConfig.CACertificateRefs { + // Default to ConfigMap if kind is not specified + kind := gatewayv1.Kind("ConfigMap") + if ref.Kind != nil { + kind = gatewayv1.Kind(*ref.Kind) + } + // Default to empty group (core API) + group := gatewayv1.Group("") + if ref.Group != nil { + group = gatewayv1.Group(*ref.Group) + } + certRef := gatewayv1.LocalObjectReference{ + Group: group, + Kind: kind, + Name: gatewayv1.ObjectName(ref.Name), + } + caCertRefs[i] = certRef + } + } + + policy := &gatewayv1.BackendTLSPolicy{ + ObjectMeta: metav1.ObjectMeta{ + Name: solrCloud.CommonBackendTLSPolicyName(), + Namespace: solrCloud.GetNamespace(), + Labels: labels, + }, + Spec: gatewayv1.BackendTLSPolicySpec{ + TargetRefs: []gatewayv1.LocalPolicyTargetReferenceWithSectionName{ + { + LocalPolicyTargetReference: gatewayv1.LocalPolicyTargetReference{ + Group: "", + Kind: "Service", + Name: gatewayv1.ObjectName(solrCloud.CommonServiceName()), + }, + }, + }, + Validation: gatewayv1.BackendTLSPolicyValidation{ + Hostname: gatewayv1.PreciseHostname(fqdn), + }, + }, + } + + // Set CA certificates or well-known CAs + if caCertRefs != nil { + policy.Spec.Validation.CACertificateRefs = caCertRefs + } else if backendTLSConfig.WellKnownCACertificates != nil { + wellKnown := gatewayv1.WellKnownCACertificatesType(*backendTLSConfig.WellKnownCACertificates) + policy.Spec.Validation.WellKnownCACertificates = &wellKnown + } + + return policy +} + +// GenerateNodeBackendTLSPolicy creates a BackendTLSPolicy for individual Solr node service +func GenerateNodeBackendTLSPolicy(solrCloud *solr.SolrCloud, nodeName string) *gatewayv1.BackendTLSPolicy { + if solrCloud.Spec.SolrAddressability.External.Gateway == nil || + solrCloud.Spec.SolrAddressability.External.Gateway.BackendTLSPolicy == nil { + return nil + } + + // Get the full FQDN for hostname validation + domainName := solrCloud.Spec.SolrAddressability.External.DomainName + fqdn := solrCloud.ExternalNodeUrl(nodeName, domainName, false) + + labels := solrCloud.SharedLabelsWith(solrCloud.GetLabels()) + backendTLSConfig := solrCloud.Spec.SolrAddressability.External.Gateway.BackendTLSPolicy + + // Convert CA certificate refs + var caCertRefs []gatewayv1.LocalObjectReference + if backendTLSConfig.CACertificateRefs != nil { + caCertRefs = make([]gatewayv1.LocalObjectReference, len(backendTLSConfig.CACertificateRefs)) + for i, ref := range backendTLSConfig.CACertificateRefs { + // Default to ConfigMap if kind is not specified + kind := gatewayv1.Kind("ConfigMap") + if ref.Kind != nil { + kind = gatewayv1.Kind(*ref.Kind) + } + // Default to empty group (core API) + group := gatewayv1.Group("") + if ref.Group != nil { + group = gatewayv1.Group(*ref.Group) + } + certRef := gatewayv1.LocalObjectReference{ + Group: group, + Kind: kind, + Name: gatewayv1.ObjectName(ref.Name), + } + caCertRefs[i] = certRef + } + } + + policy := &gatewayv1.BackendTLSPolicy{ + ObjectMeta: metav1.ObjectMeta{ + Name: solrCloud.NodeBackendTLSPolicyName(nodeName), + Namespace: solrCloud.GetNamespace(), + Labels: labels, + }, + Spec: gatewayv1.BackendTLSPolicySpec{ + TargetRefs: []gatewayv1.LocalPolicyTargetReferenceWithSectionName{ + { + LocalPolicyTargetReference: gatewayv1.LocalPolicyTargetReference{ + Group: "", + Kind: "Service", + Name: gatewayv1.ObjectName(nodeName), + }, + }, + }, + Validation: gatewayv1.BackendTLSPolicyValidation{ + Hostname: gatewayv1.PreciseHostname(fqdn), + }, + }, + } + + // Set CA certificates or well-known CAs + if caCertRefs != nil { + policy.Spec.Validation.CACertificateRefs = caCertRefs + } else if backendTLSConfig.WellKnownCACertificates != nil { + wellKnown := gatewayv1.WellKnownCACertificatesType(*backendTLSConfig.WellKnownCACertificates) + policy.Spec.Validation.WellKnownCACertificates = &wellKnown + } + + return policy +} + +// CopyBackendTLSPolicyFields copies fields from one BackendTLSPolicy to another +func CopyBackendTLSPolicyFields(from, to *gatewayv1.BackendTLSPolicy, logger logr.Logger) bool { + requireUpdate := false + + if !DeepEqualWithNils(to.Labels, from.Labels) { + logger.Info("BackendTLSPolicy labels have changed") + requireUpdate = true + to.Labels = from.Labels + } + + if !DeepEqualWithNils(to.Spec.TargetRefs, from.Spec.TargetRefs) { + logger.Info("BackendTLSPolicy targetRefs have changed") + requireUpdate = true + to.Spec.TargetRefs = from.Spec.TargetRefs + } + + if !DeepEqualWithNils(to.Spec.Validation, from.Spec.Validation) { + logger.Info("BackendTLSPolicy validation have changed") + requireUpdate = true + to.Spec.Validation = from.Spec.Validation + } + + return requireUpdate +} diff --git a/dependency_licenses.csv b/dependency_licenses.csv index 11a57718..c73ce78f 100644 --- a/dependency_licenses.csv +++ b/dependency_licenses.csv @@ -69,6 +69,7 @@ k8s.io/utils,https://github.com/kubernetes/utils/blob/a95e086a2553/LICENSE,Apach k8s.io/utils/internal/third_party/forked/golang,https://github.com/kubernetes/utils/blob/a95e086a2553/internal/third_party/forked/golang/LICENSE,BSD-3-Clause k8s.io/utils/third_party/forked/golang/btree,https://github.com/kubernetes/utils/blob/a95e086a2553/third_party/forked/golang/btree/LICENSE,Apache-2.0 sigs.k8s.io/controller-runtime,https://github.com/kubernetes-sigs/controller-runtime/blob/v0.24.1/LICENSE,Apache-2.0 +sigs.k8s.io/gateway-api/apis/v1,https://github.com/kubernetes-sigs/gateway-api/blob/v1.6.1/LICENSE,Apache-2.0 sigs.k8s.io/json,https://github.com/kubernetes-sigs/json/blob/2d320260d730/LICENSE,Apache-2.0 sigs.k8s.io/json,https://github.com/kubernetes-sigs/json/blob/2d320260d730/LICENSE,BSD-3-Clause sigs.k8s.io/randfill,https://github.com/kubernetes-sigs/randfill/blob/v1.0.0/LICENSE,Apache-2.0 diff --git a/docs/modules/solr-cloud/nav.adoc b/docs/modules/solr-cloud/nav.adoc index c16f8a12..62ee0361 100644 --- a/docs/modules/solr-cloud/nav.adoc +++ b/docs/modules/solr-cloud/nav.adoc @@ -19,6 +19,7 @@ * xref:index.adoc[Overview] * xref:solr-cloud-crd.adoc[] ** xref:addressability.adoc[] +*** xref:gateway-api.adoc[Gateway API] ** xref:zookeeper.adoc[] ** xref:custom-solr-config.adoc[Custom Solr Configuration] ** xref:tls.adoc[] diff --git a/docs/modules/solr-cloud/pages/addressability.adoc b/docs/modules/solr-cloud/pages/addressability.adoc index 158a63de..6b7dfe40 100644 --- a/docs/modules/solr-cloud/pages/addressability.adoc +++ b/docs/modules/solr-cloud/pages/addressability.adoc @@ -25,7 +25,7 @@ Under `SolrCloud.Spec.solrAddressability`: * **`kubeDomain`** - Specifies an override of the default Kubernetes cluster domain name, `cluster.local`. This option should only be used if the Kubernetes cluster has been setup with a custom domain name. * **`external`** - Expose the cloud externally, outside of the kubernetes cluster in which it is running. ** **`method`** - (Required) The method by which your cloud will be exposed externally. -Currently available options are https://kubernetes.io/docs/concepts/services-networking/ingress/[`Ingress`] and https://github.com/kubernetes-sigs/external-dns[`ExternalDNS`]. +Currently available options are https://kubernetes.io/docs/concepts/services-networking/ingress/[`Ingress`], the xref:gateway-api.adoc[`Gateway`] API, and https://github.com/kubernetes-sigs/external-dns[`ExternalDNS`]. The goal is to support more methods in the future, such as LoadBalanced Services. ** **`domainName`** - (Required) The primary domain name to open your cloud endpoints on. If `useExternalAddress` is set to `true`, then this is the domain that will be used in Solr Node names. ** **`additionalDomainNames`** - You can choose to listen on additional domains for each endpoint, however Solr will not register itself under these names. diff --git a/docs/modules/solr-cloud/pages/gateway-api.adoc b/docs/modules/solr-cloud/pages/gateway-api.adoc new file mode 100644 index 00000000..faa41916 --- /dev/null +++ b/docs/modules/solr-cloud/pages/gateway-api.adoc @@ -0,0 +1,272 @@ += Gateway API +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +== Overview + +The Solr Operator supports using the https://gateway-api.sigs.k8s.io/[Kubernetes Gateway API] for external addressability of SolrClouds. +Gateway API is a vendor-neutral, Kubernetes-native API for managing ingress traffic and is the successor to the Ingress API. + +When you configure `spec.solrAddressability.external.method: Gateway`, the Solr Operator creates and manages HTTPRoute resources +that route external traffic to your Solr nodes through an existing Gateway resource in your cluster. + +== What Gets Created + +When Gateway mode is enabled, the Solr Operator automatically creates the following Kubernetes resources: + +=== HTTPRoute Resources + +The operator creates HTTPRoute resources to route traffic to Solr: + +* **Common HTTPRoute**: Routes traffic to the common Solr service (load-balanced across all nodes) +** Named: `-solrcloud-common` +** Hostname: `--solrcloud.` +* **Per-Node HTTPRoutes**: Routes traffic directly to individual Solr nodes (when `hideNodes: false`) +** Named: `-solrcloud-` +** Hostname: `--solrcloud-.` + +All HTTPRoutes are owned by the SolrCloud resource and will be automatically cleaned up when the SolrCloud is deleted. + +=== Services + +The same services are created as with other external addressability methods, depending on whether per-node Services are required: + +* Common service (load-balanced) +* Headless service (when not using per-node services) +* Per-node services (when individual node access is enabled) + +== What the Operator Assumes + +The Gateway mode assumes the following resources already exist in your cluster: + +. **Gateway API CRDs**: The Gateway API CRDs must be installed in your cluster +** Minimum version: v1.4.0 (required for `BackendTLSPolicy` support) +** Required CRDs: `Gateway`, `GatewayClass`, `HTTPRoute`, `BackendTLSPolicy` (optional, for TLS backends) +. **Gateway Resource**: A Gateway resource must already exist and be managed by your platform team +** The operator only manages HTTPRoute resources, not the Gateway itself +** The Gateway must be configured with appropriate listeners and TLS termination (if needed) +. **Gateway Controller**: A Gateway controller implementation must be running (e.g., NGINX Gateway Fabric, Istio, Envoy Gateway) + +The Solr Operator does **not** create or manage Gateway or GatewayClass resources - these are infrastructure-level resources +typically managed by platform administrators and are specific to the Gateway implementation deployed in your cluster (e.g., NGINX Gateway Fabric, Istio, Envoy Gateway). + +=== CRD Detection + +Gateway support is enabled automatically; there is no flag to turn it on or off. +At startup, the operator queries the Kubernetes API server to determine whether the Gateway API CRDs are installed. +Gateway support is enabled only when **both** the `HTTPRoute` and `BackendTLSPolicy` types (API group `gateway.networking.k8s.io/v1`) are registered. +If either is missing, the operator logs that Gateway support is disabled, does not watch these types, and ignores `method: Gateway` configurations. + +IMPORTANT: Detection happens only once, when the operator starts. If you install (or remove) the Gateway API CRDs while the operator is running, you must restart the operator pod for the change to take effect. + +== Configuration + +Configure Gateway mode in your SolrCloud spec: + +[source,yaml] +---- +apiVersion: solr.apache.org/v1beta1 +kind: SolrCloud +metadata: + name: example + namespace: solr-ns +spec: + replicas: 3 + solrImage: + tag: "9.7.0" + solrAddressability: + external: + method: Gateway + domainName: solr.example.com + useExternalAddress: true + gateway: + # Reference to existing Gateway resource(s) + parentRefs: + - name: my-gateway + namespace: gateway-ns + sectionName: https # Optional: specific listener name + + # Optional: annotations to add to HTTPRoute resources + annotations: + example.com/custom-annotation: "value" + + # Optional: labels to add to HTTPRoute resources + labels: + app: solr + environment: production +---- + +=== Configuration Options + +* **`parentRefs`** (required): List of Gateway resources to attach HTTPRoutes to +** `name`: Name of the Gateway resource +** `namespace`: Namespace of the Gateway (can be different from SolrCloud namespace) +** `sectionName`: Optional listener name within the Gateway +* **`annotations`**: Optional annotations to add to all created HTTPRoute resources +* **`labels`**: Optional labels to add to all created HTTPRoute resources +* **`domainName`**: Base domain for constructing hostnames +* **`useExternalAddress`**: When true, Solr nodes use external addresses for inter-node communication +* **`hideCommon`**: Set to true to skip creating the common HTTPRoute (default: false) +* **`hideNodes`**: Set to true to skip creating per-node HTTPRoutes (default: false) + +== Backend TLS Policy + +When using TLS-enabled Solr (`spec.solrTLS` is configured), the Solr Operator automatically sets `appProtocol: https` on all Services. +The operator also supports creating `BackendTLSPolicy` resources to configure secure connections between the Gateway and Solr backend services. + +=== Configuring BackendTLSPolicy + +The Solr Operator can automatically create and manage `BackendTLSPolicy` resources (Gateway API v1) when configured in the SolrCloud spec: + +[source,yaml] +---- +apiVersion: solr.apache.org/v1beta1 +kind: SolrCloud +metadata: + name: example + namespace: solr-ns +spec: + replicas: 3 + solrImage: + tag: "9.7.0" + # Enable TLS for Solr + solrTLS: + pkcs12Secret: + name: solr-tls-cert + key: keystore.p12 + solrAddressability: + external: + method: Gateway + domainName: solr.example.com + gateway: + parentRefs: + - name: my-gateway + namespace: gateway-ns + # Configure BackendTLSPolicy for secure backend connections + backendTLSPolicy: + # Option 1: Reference CA certificate from a ConfigMap (default) + caCertificateRefs: + - name: solr-ca-cert + # kind: ConfigMap # Optional, defaults to ConfigMap + # group: "" # Optional, defaults to "" (core API) + + # Option 2: Use well-known CA certificates + # wellKnownCACertificates: "System" +---- + +The generated `BackendTLSPolicy` will look like: + +[source,yaml] +---- +apiVersion: gateway.networking.k8s.io/v1 +kind: BackendTLSPolicy +metadata: + name: example-solrcloud-common + namespace: solr-ns +spec: + targetRefs: + - name: example-solrcloud-common + kind: Service + group: "" + validation: + hostname: example-solrcloud-common + caCertificateRefs: + - group: "" + kind: ConfigMap + name: solr-ca-cert +---- + +=== BackendTLSPolicy Options + +The `backendTLSPolicy` field supports two mutually exclusive options: + +* **`caCertificateRefs`**: References to Kubernetes ConfigMaps or Secrets containing CA certificates +** `name`: Name of the ConfigMap or Secret (required) +** `kind`: Resource kind (optional, defaults to "ConfigMap") +** `group`: API group (optional, defaults to "" for core API) +** Maximum of 8 certificate references +** **Note**: ConfigMaps must contain the CA certificate in a key named `ca.crt` +* **`wellKnownCACertificates`**: Use system CA certificates (e.g., "System") +** Only one of `caCertificateRefs` or `wellKnownCACertificates` can be specified + +When configured, the operator creates: + +* `BackendTLSPolicy` for the common service (if not hidden) +* `BackendTLSPolicy` for each node service (if not hidden) + +These policies configure the Gateway to validate backend TLS certificates and establish secure connections to Solr pods. + +=== Gateway Implementation Support + +NOTE: `BackendTLSPolicy` is part of the Gateway API standard (GA in v1.4.0+), but support varies by implementation. + +[cols="1,1",options="header"] +|=== +| Gateway Implementation | BackendTLSPolicy Support + +| **Standard Gateway API** | ✅ v1 (GA as of v1.4.0) +| **Envoy Gateway** | ✅ Full support +| **kgateway** | ✅ Full support +| **Istio** | ⚠️ Use `DestinationRule` instead +| **NGINX Gateway Fabric** | ✅ Supported +| **GKE Gateway** | ⚠️ Automatic via `appProtocol` +|=== + +**Requirements**: Gateway API v1.4.0 or later is required for `BackendTLSPolicy` support. + +Refer to your Gateway implementation's documentation for specific backend TLS configuration requirements. + +== Complete Example + +[source,yaml] +---- +apiVersion: solr.apache.org/v1beta1 +kind: SolrCloud +metadata: + name: example + namespace: solr-ns +spec: + replicas: 3 + solrImage: + tag: "9.7.0" + solrAddressability: + external: + method: Gateway + domainName: solr.example.com + useExternalAddress: true + gateway: + parentRefs: + - name: my-gateway + namespace: gateway-ns + annotations: + example.com/rate-limit: "1000" + labels: + app: solr +---- + +This configuration will create: + +* HTTPRoute: `example-solrcloud-common` → `solr-ns-example-solrcloud.solr.example.com` +* HTTPRoute: `example-solrcloud-0` → `solr-ns-example-solrcloud-0.solr.example.com` +* HTTPRoute: `example-solrcloud-1` → `solr-ns-example-solrcloud-1.solr.example.com` +* HTTPRoute: `example-solrcloud-2` → `solr-ns-example-solrcloud-2.solr.example.com` + +== References + +* https://gateway-api.sigs.k8s.io/[Gateway API Documentation] +* https://gateway-api.sigs.k8s.io/geps/gep-1897/[BackendTLSPolicy (GEP-1897)] +* https://kubernetes.io/docs/concepts/services-networking/service/#application-protocol[Kubernetes Service appProtocol] diff --git a/go.mod b/go.mod index bd9f67d7..bcf10748 100644 --- a/go.mod +++ b/go.mod @@ -19,6 +19,7 @@ require ( k8s.io/client-go v0.36.2 k8s.io/utils v0.0.0-20260617174310-a95e086a2553 sigs.k8s.io/controller-runtime v0.24.1 + sigs.k8s.io/gateway-api v1.6.1 ) require ( @@ -144,7 +145,6 @@ require ( k8s.io/kubectl v0.36.2 // indirect k8s.io/streaming v0.36.2 // indirect oras.land/oras-go/v2 v2.6.1 // indirect - sigs.k8s.io/gateway-api v1.5.1 // indirect sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 // indirect sigs.k8s.io/kustomize/api v0.21.1 // indirect sigs.k8s.io/kustomize/kyaml v0.21.1 // indirect diff --git a/go.sum b/go.sum index 36817b47..84aaf664 100644 --- a/go.sum +++ b/go.sum @@ -463,8 +463,8 @@ oras.land/oras-go/v2 v2.6.1 h1:bonOEkjLfp8tt6qXWRRWP6p1F+9octchOf2EqnWB4Zs= oras.land/oras-go/v2 v2.6.1/go.mod h1:dhtFrFOuZuDtAVeZ9FUnaa5zfzplG3ZnFX9/uH1J/Yk= sigs.k8s.io/controller-runtime v0.24.1 h1:miPEwrmirImAvgME1L9qebGHrOnGJoVmVdtOU9fRfo4= sigs.k8s.io/controller-runtime v0.24.1/go.mod h1:vFkfY5fGt5xAC/sKb8IBFKgWPNKG9OUG29dR8Y2wImw= -sigs.k8s.io/gateway-api v1.5.1 h1:RqVRIlkhLhUO8wOHKTLnTJA6o/1un4po4/6M1nRzdd0= -sigs.k8s.io/gateway-api v1.5.1/go.mod h1:GvCETiaMAlLym5CovLxGjS0NysqFk3+Yuq3/rh6QL2o= +sigs.k8s.io/gateway-api v1.6.1 h1:mock6phZbI6rvZerwrVNk7hVNymQgHo+6sJ81Ia7ftY= +sigs.k8s.io/gateway-api v1.6.1/go.mod h1:FVfx3t389ybeXOqvDghLbdvJdSCfI/PReqCUI3lu3mY= sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 h1:IpInykpT6ceI+QxKBbEflcR5EXP7sU1kvOlxwZh5txg= sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg= sigs.k8s.io/kustomize/api v0.21.1 h1:lzqbzvz2CSvsjIUZUBNFKtIMsEw7hVLJp0JeSIVmuJs= diff --git a/helm/solr-operator/crds/crds.yaml b/helm/solr-operator/crds/crds.yaml index 45a5b4fe..e9a3d966 100644 --- a/helm/solr-operator/crds/crds.yaml +++ b/helm/solr-operator/crds/crds.yaml @@ -10967,6 +10967,115 @@ spec: For the LoadBalancer method, this field is optional and will only be used when useExternalAddress=true. If used with the LoadBalancer method, you will need DNS routing to the LoadBalancer IP address through the url template given above. type: string + gateway: + description: |- + Gateway defines settings for Kubernetes Gateway API routing. + + This option is only available when Method=Gateway. + The referenced Gateway must already exist and be managed by your platform team. + The Solr Operator only manages the HTTPRoute resources. + properties: + additionalHostnames: + description: |- + AdditionalHostnames specifies extra hostnames to include in the common HTTPRoute. + These are appended to the auto-generated hostnames derived from DomainName and AdditionalDomainNames. + This is useful for adding alias hostnames that should also route to the common Solr service. + items: + type: string + maxItems: 16 + type: array + annotations: + additionalProperties: + type: string + description: Annotations to add to HTTPRoute resources + type: object + backendTLSPolicy: + description: |- + BackendTLSPolicy defines TLS configuration for backend connections from Gateway to Solr pods. + + This is used when Solr pods are running with TLS enabled (spec.solrTLS) and the Gateway + needs to establish secure connections to the backend services. + + The Solr Operator will create BackendTLSPolicy resources for each HTTPRoute. + maxProperties: 1 + properties: + caCertificateRefs: + description: |- + CACertificateRefs contains one or more references to Kubernetes objects that contain + TLS certificates of the Certificate Authorities that can be used as a trust anchor + to validate the certificates presented by the backend. + + If specified, WellKnownCACertificates must not be set. + items: + description: GatewayCertificateReference identifies + a certificate object in Kubernetes + properties: + group: + description: Group of the resource + type: string + kind: + default: ConfigMap + description: Kind of the resource (e.g., "ConfigMap" + or "Secret") + type: string + name: + description: Name of the Kubernetes resource + (e.g., ConfigMap or Secret) + type: string + required: + - name + type: object + maxItems: 8 + type: array + wellKnownCACertificates: + description: |- + WellKnownCACertificates specifies whether system CA certificates may be used in the + TLS handshake between the gateway and backend pod. + + If WellKnownCACertificates is unspecified or empty (""), then CACertificateRefs must be + specified with at least one entry for a valid configuration. + + Only one of CACertificateRefs or WellKnownCACertificates may be specified, not both. + If specified, CACertificateRefs must not be set. + type: string + type: object + labels: + additionalProperties: + type: string + description: Labels to add to HTTPRoute resources + type: object + parentRefs: + description: |- + ParentRefs specifies the Gateway(s) to attach HTTPRoutes to. + This is required when using method=Gateway. + + The referenced Gateway must already exist and be managed by your platform team. + The Solr Operator only manages the HTTPRoute resources. + items: + description: GatewayParentReference identifies a parent + Gateway resource to attach HTTPRoutes to + properties: + name: + description: Name of the Gateway resource + type: string + namespace: + description: |- + Namespace of the Gateway resource. + If not specified, defaults to the HTTPRoute's namespace. + type: string + sectionName: + description: |- + SectionName refers to a specific listener on the Gateway. + For example, "https" or "http". + type: string + required: + - name + type: object + minItems: 1 + type: array + required: + - parentRefs + type: object hideCommon: description: |- Do not expose the common Solr service externally. This affects a single service. @@ -11006,6 +11115,7 @@ spec: enum: - Ingress - ExternalDNS + - Gateway type: string nodePortOverride: description: |- diff --git a/helm/solr-operator/templates/role.yaml b/helm/solr-operator/templates/role.yaml index 861ad0c7..d3fb9596 100644 --- a/helm/solr-operator/templates/role.yaml +++ b/helm/solr-operator/templates/role.yaml @@ -103,6 +103,26 @@ rules: - statefulsets/status verbs: - get +- apiGroups: + - gateway.networking.k8s.io + resources: + - backendtlspolicies + - httproutes + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - gateway.networking.k8s.io + resources: + - backendtlspolicies/status + - httproutes/status + verbs: + - get - apiGroups: - networking.k8s.io resources: diff --git a/main.go b/main.go index 27003dfd..bba1b2ff 100644 --- a/main.go +++ b/main.go @@ -22,23 +22,26 @@ import ( "crypto/x509" "flag" "fmt" - "github.com/apache/solr-operator/controllers/util/solr_api" - "github.com/apache/solr-operator/version" - "github.com/fsnotify/fsnotify" - zkApi "github.com/pravega/zookeeper-operator/api/v1beta1" "io/ioutil" "net/http" "os" "path/filepath" "runtime" + "strings" + + "github.com/apache/solr-operator/controllers/util/solr_api" + "github.com/apache/solr-operator/version" + "github.com/fsnotify/fsnotify" + zkApi "github.com/pravega/zookeeper-operator/api/v1beta1" "sigs.k8s.io/controller-runtime/pkg/cache" metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" - "strings" + gatewayv1 "sigs.k8s.io/gateway-api/apis/v1" // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) // to ensure that exec-entrypoint and run can make use of them. _ "k8s.io/client-go/plugin/pkg/client/auth" + "k8s.io/apimachinery/pkg/api/meta" k8sRuntime "k8s.io/apimachinery/pkg/runtime" utilruntime "k8s.io/apimachinery/pkg/util/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" @@ -86,6 +89,8 @@ func init() { utilruntime.Must(solrv1beta1.AddToScheme(scheme)) utilruntime.Must(zkApi.AddToScheme(scheme)) + + utilruntime.Must(gatewayv1.AddToScheme(scheme)) //+kubebuilder:scaffold:scheme flag.BoolVar(&useZookeeperCRD, "zk-operator", true, "The operator will not use the zk operator & crd when this flag is set to false.") @@ -181,6 +186,16 @@ func main() { controllers.UseZkCRD(useZookeeperCRD) + // Detect whether the Gateway API CRDs are installed in the cluster. Both the HTTPRoute + // and BackendTLSPolicy types must be present for the operator to manage Gateway resources. + useGatewayAPI := gatewayAPIInstalled(mgr.GetRESTMapper()) + if useGatewayAPI { + setupLog.Info("Gateway API CRDs detected; enabling Gateway support") + } else { + setupLog.Info("Gateway API CRDs not detected; Gateway support disabled") + } + controllers.UseGatewayAPI(useGatewayAPI) + // watch TLS files for update if clientCertPath != "" { var watcher *fsnotify.Watcher @@ -348,3 +363,18 @@ func buildTLSTransport() (*http.Transport, error) { func getClientCertificate(*tls.CertificateRequestInfo) (*tls.Certificate, error) { return clientCertificate, nil } + +// gatewayAPIInstalled reports whether the Gateway API CRDs required by the operator +// (both HTTPRoute and BackendTLSPolicy) are registered with the API server. +func gatewayAPIInstalled(mapper meta.RESTMapper) bool { + for _, kind := range []string{"HTTPRoute", "BackendTLSPolicy"} { + gvk := gatewayv1.SchemeGroupVersion.WithKind(kind) + if _, err := mapper.RESTMapping(gvk.GroupKind(), gvk.Version); err != nil { + if !meta.IsNoMatchError(err) { + setupLog.Error(err, "Error checking for Gateway API CRD; disabling Gateway support", "kind", kind) + } + return false + } + } + return true +} diff --git a/tests/e2e/resource_utils_backendtls_test.go b/tests/e2e/resource_utils_backendtls_test.go new file mode 100644 index 00000000..c1e25f80 --- /dev/null +++ b/tests/e2e/resource_utils_backendtls_test.go @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package e2e + +import ( + . "github.com/onsi/gomega" + "golang.org/x/net/context" + "sigs.k8s.io/controller-runtime/pkg/client" + gatewayv1 "sigs.k8s.io/gateway-api/apis/v1" +) + +func expectBackendTLSPolicy(ctx context.Context, parentResource client.Object, policyName string, additionalOffset ...int) *gatewayv1.BackendTLSPolicy { + return expectBackendTLSPolicyWithChecks(ctx, parentResource, policyName, nil, resolveOffset(additionalOffset)) +} + +func expectBackendTLSPolicyWithChecks(ctx context.Context, parentResource client.Object, policyName string, additionalChecks func(Gomega, *gatewayv1.BackendTLSPolicy), additionalOffset ...int) *gatewayv1.BackendTLSPolicy { + policy := &gatewayv1.BackendTLSPolicy{} + EventuallyWithOffset(resolveOffset(additionalOffset), func(g Gomega) { + g.Expect(k8sClient.Get(ctx, resourceKey(parentResource, policyName), policy)).To(Succeed(), "Expected BackendTLSPolicy does not exist") + + if additionalChecks != nil { + additionalChecks(g, policy) + } + }).Should(Succeed()) + return policy +} + +func expectNoBackendTLSPolicy(ctx context.Context, parentResource client.Object, policyName string, additionalOffset ...int) { + ConsistentlyWithOffset(resolveOffset(additionalOffset), func() error { + return k8sClient.Get(ctx, resourceKey(parentResource, policyName), &gatewayv1.BackendTLSPolicy{}) + }).Should(MatchError("backendtlspolicies.gateway.networking.k8s.io \""+policyName+"\" not found"), "BackendTLSPolicy exists when it should not") +} + +func eventuallyExpectNoBackendTLSPolicy(ctx context.Context, parentResource client.Object, policyName string, additionalOffset ...int) { + EventuallyWithOffset(resolveOffset(additionalOffset), func() error { + return k8sClient.Get(ctx, resourceKey(parentResource, policyName), &gatewayv1.BackendTLSPolicy{}) + }).Should(MatchError("backendtlspolicies.gateway.networking.k8s.io \""+policyName+"\" not found"), "BackendTLSPolicy exists when it should not") +} diff --git a/tests/e2e/resource_utils_test.go b/tests/e2e/resource_utils_test.go index bef49efa..c72a9b55 100644 --- a/tests/e2e/resource_utils_test.go +++ b/tests/e2e/resource_utils_test.go @@ -37,6 +37,7 @@ import ( "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/intstr" "sigs.k8s.io/controller-runtime/pkg/client" + gatewayv1 "sigs.k8s.io/gateway-api/apis/v1" ) // Add one to an optional offset @@ -500,6 +501,47 @@ func eventuallyExpectNoIngress(ctx context.Context, parentResource client.Object }).Should(MatchError("ingresses.networking.k8s.io \""+ingressName+"\" not found"), "Ingress exists when it should not") } +func expectHTTPRoute(ctx context.Context, parentResource client.Object, httpRouteName string, additionalOffset ...int) *gatewayv1.HTTPRoute { + return expectHTTPRouteWithChecks(ctx, parentResource, httpRouteName, nil, resolveOffset(additionalOffset)) +} + +func expectHTTPRouteWithChecks(ctx context.Context, parentResource client.Object, httpRouteName string, additionalChecks func(Gomega, *gatewayv1.HTTPRoute), additionalOffset ...int) *gatewayv1.HTTPRoute { + httpRoute := &gatewayv1.HTTPRoute{} + EventuallyWithOffset(resolveOffset(additionalOffset), func(g Gomega) { + g.Expect(k8sClient.Get(ctx, resourceKey(parentResource, httpRouteName), httpRoute)).To(Succeed(), "Expected HTTPRoute does not exist") + + if additionalChecks != nil { + additionalChecks(g, httpRoute) + } + }).Should(Succeed()) + return httpRoute +} + +func expectHTTPRouteWithConsistentChecks(ctx context.Context, parentResource client.Object, httpRouteName string, additionalChecks func(Gomega, *gatewayv1.HTTPRoute), additionalOffset ...int) *gatewayv1.HTTPRoute { + httpRoute := &gatewayv1.HTTPRoute{} + ConsistentlyWithOffset(resolveOffset(additionalOffset), func(g Gomega) { + g.Expect(k8sClient.Get(ctx, resourceKey(parentResource, httpRouteName), httpRoute)).To(Succeed(), "Expected HTTPRoute does not exist") + + if additionalChecks != nil { + additionalChecks(g, httpRoute) + } + }).Should(Succeed()) + + return httpRoute +} + +func expectNoHTTPRoute(ctx context.Context, parentResource client.Object, httpRouteName string, additionalOffset ...int) { + ConsistentlyWithOffset(resolveOffset(additionalOffset), func() error { + return k8sClient.Get(ctx, resourceKey(parentResource, httpRouteName), &gatewayv1.HTTPRoute{}) + }).Should(MatchError("httproutes.gateway.networking.k8s.io \""+httpRouteName+"\" not found"), "HTTPRoute exists when it should not") +} + +func eventuallyExpectNoHTTPRoute(ctx context.Context, parentResource client.Object, httpRouteName string, additionalOffset ...int) { + EventuallyWithOffset(resolveOffset(additionalOffset), func() error { + return k8sClient.Get(ctx, resourceKey(parentResource, httpRouteName), &gatewayv1.HTTPRoute{}) + }).Should(MatchError("httproutes.gateway.networking.k8s.io \""+httpRouteName+"\" not found"), "HTTPRoute exists when it should not") +} + func expectPodDisruptionBudget(ctx context.Context, parentResource client.Object, podDisruptionBudgetName string, selector *metav1.LabelSelector, maxUnavailable intstr.IntOrString, additionalOffset ...int) *policyv1.PodDisruptionBudget { return expectPodDisruptionBudgetWithChecks(ctx, parentResource, podDisruptionBudgetName, selector, maxUnavailable, nil, resolveOffset(additionalOffset)) } diff --git a/tests/e2e/solrcloud_gateway_test.go b/tests/e2e/solrcloud_gateway_test.go new file mode 100644 index 00000000..2988341a --- /dev/null +++ b/tests/e2e/solrcloud_gateway_test.go @@ -0,0 +1,282 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package e2e + +import ( + "context" + solrv1beta1 "github.com/apache/solr-operator/api/v1beta1" + "github.com/apache/solr-operator/controllers/util" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + gatewayv1 "sigs.k8s.io/gateway-api/apis/v1" +) + +var _ = Describe("E2E - SolrCloud - Gateway API", func() { + var ( + solrCloud *solrv1beta1.SolrCloud + gatewayNamespace = "default" + gatewayName = "test-gateway" + ) + + BeforeEach(func() { + solrCloud = generateBaseSolrCloud(1) + solrCloud.Spec.SolrAddressability = solrv1beta1.SolrAddressabilityOptions{ + External: &solrv1beta1.ExternalAddressability{ + Method: solrv1beta1.Gateway, + UseExternalAddress: true, + DomainName: testDomain, + Gateway: &solrv1beta1.SolrGatewayOptions{ + ParentRefs: []solrv1beta1.GatewayParentReference{ + { + Name: gatewayName, + Namespace: &gatewayNamespace, + }, + }, + }, + }, + } + }) + + JustBeforeEach(func(ctx context.Context) { + By("creating the SolrCloud") + Expect(k8sClient.Create(ctx, solrCloud)).To(Succeed()) + + DeferCleanup(func(ctx context.Context) { + cleanupTest(ctx, solrCloud) + }) + + By("Waiting for the SolrCloud to come up healthy") + solrCloud = expectSolrCloudToBeReady(ctx, solrCloud) + + By("creating a first Solr Collection") + createAndQueryCollection(ctx, solrCloud, "basic", 1, 1) + }) + + Context("Can Remove HTTPRoutes and Services when changing addressability", func() { + + It("Can adapt to changing needs", func(ctx context.Context) { + By("testing the Solr StatefulSet") + statefulSet := expectStatefulSet(ctx, solrCloud, solrCloud.StatefulSetName()) + // Pod Annotations test + Expect(statefulSet.Spec.Template.Annotations).To(HaveKeyWithValue(util.ServiceTypeAnnotation, util.PerNodeServiceType), "Since external address is used for advertising, the perNode service should be specified in the pod annotations.") + + By("testing the Solr Common Service") + expectService(ctx, solrCloud, solrCloud.CommonServiceName(), statefulSet.Spec.Selector.MatchLabels, false) + + By("ensuring the Solr Headless Service does not exist") + expectNoService(ctx, solrCloud, solrCloud.HeadlessServiceName(), "Headless service shouldn't exist, but it does.") + + By("making sure the individual Solr Node Services exist and route correctly") + nodeNames := solrCloud.GetAllSolrPodNames() + Expect(nodeNames).To(HaveLen(1), "SolrCloud has incorrect number of nodeNames.") + for _, nodeName := range nodeNames { + expectService(ctx, solrCloud, nodeName, util.MergeLabelsOrAnnotations(statefulSet.Spec.Selector.MatchLabels, map[string]string{"statefulset.kubernetes.io/pod-name": nodeName}), false) + } + + By("making sure Common HTTPRoute was created correctly") + expectHTTPRoute(ctx, solrCloud, solrCloud.CommonHTTPRouteName()) + + By("making sure Node HTTPRoutes were created correctly") + for _, nodeName := range nodeNames { + expectHTTPRoute(ctx, solrCloud, solrCloud.NodeHTTPRouteName(nodeName)) + } + + By("Turning off node external addressability and making sure the node services are deleted") + expectSolrCloudWithChecks(ctx, solrCloud, func(g Gomega, found *solrv1beta1.SolrCloud) { + found.Spec.SolrAddressability.External.HideNodes = true + found.Spec.SolrAddressability.External.UseExternalAddress = false + g.Expect(k8sClient.Update(ctx, found)).To(Succeed(), "Couldn't update the solrCloud to not advertise the nodes externally.") + }) + + // Since node external addressability is off, but common external addressability is on, the common HTTPRoute should exist, but the node HTTPRoutes should not + expectHTTPRoute(ctx, solrCloud, solrCloud.CommonHTTPRouteName()) + + expectService(ctx, solrCloud, solrCloud.HeadlessServiceName(), statefulSet.Spec.Selector.MatchLabels, true) + + for _, nodeName := range nodeNames { + eventuallyExpectNoHTTPRoute(ctx, solrCloud, solrCloud.NodeHTTPRouteName(nodeName)) + } + pod := expectPodNow(ctx, solrCloud, solrCloud.GetSolrPodName(0)) + Expect(pod.Annotations).To(HaveKeyWithValue(util.ServiceTypeAnnotation, util.HeadlessServiceType)) + + By("Turning off common external addressability and making sure the HTTPRoutes are deleted") + expectSolrCloudWithChecks(ctx, solrCloud, func(g Gomega, found *solrv1beta1.SolrCloud) { + found.Spec.SolrAddressability.External = nil + g.Expect(k8sClient.Update(ctx, found)).To(Succeed(), "Couldn't update the solrCloud to remove external addressability") + }) + eventuallyExpectNoHTTPRoute(ctx, solrCloud, solrCloud.CommonHTTPRouteName()) + + By("Turning back on common external addressability and making sure the headless service is deleted") + expectSolrCloudWithChecks(ctx, solrCloud, func(g Gomega, found *solrv1beta1.SolrCloud) { + found.Spec.SolrAddressability = solrv1beta1.SolrAddressabilityOptions{ + External: &solrv1beta1.ExternalAddressability{ + Method: solrv1beta1.Gateway, + UseExternalAddress: true, + DomainName: testDomain, + Gateway: &solrv1beta1.SolrGatewayOptions{ + ParentRefs: []solrv1beta1.GatewayParentReference{ + { + Name: gatewayName, + Namespace: &gatewayNamespace, + }, + }, + }, + }, + } + g.Expect(k8sClient.Update(ctx, found)).To(Succeed(), "Couldn't update the solrCloud to add external addressability") + }) + expectHTTPRoute(ctx, solrCloud, solrCloud.CommonHTTPRouteName()) + + By("testing the Solr Common Service") + expectService(ctx, solrCloud, solrCloud.CommonServiceName(), statefulSet.Spec.Selector.MatchLabels, false) + + By("ensuring the Solr Headless Service does not exist") + expectNoService(ctx, solrCloud, solrCloud.HeadlessServiceName(), "Headless service shouldn't exist, but it does.") + }) + }) + + Context("BackendTLSPolicy Management", func() { + var ( + caCertConfigMapName = "solr-ca-cert" + ) + + It("Creates and manages BackendTLSPolicy resources", func(ctx context.Context) { + By("verifying BackendTLSPolicy resources do not exist initially") + expectNoBackendTLSPolicy(ctx, solrCloud, solrCloud.CommonBackendTLSPolicyName()) + nodeNames := solrCloud.GetAllSolrPodNames() + for _, nodeName := range nodeNames { + expectNoBackendTLSPolicy(ctx, solrCloud, solrCloud.NodeBackendTLSPolicyName(nodeName)) + } + + By("enabling BackendTLSPolicy with CA certificate reference") + expectSolrCloudWithChecks(ctx, solrCloud, func(g Gomega, found *solrv1beta1.SolrCloud) { + found.Spec.SolrAddressability.External.Gateway.BackendTLSPolicy = &solrv1beta1.SolrBackendTLSPolicy{ + CACertificateRefs: []solrv1beta1.GatewayCertificateReference{ + { + Name: caCertConfigMapName, + }, + }, + } + g.Expect(k8sClient.Update(ctx, found)).To(Succeed(), "Couldn't update the solrCloud to add BackendTLSPolicy") + }) + + By("verifying Common BackendTLSPolicy was created correctly") + commonPolicy := expectBackendTLSPolicyWithChecks(ctx, solrCloud, solrCloud.CommonBackendTLSPolicyName(), func(g Gomega, policy *gatewayv1.BackendTLSPolicy) { + g.Expect(policy.Spec.TargetRefs).To(HaveLen(1), "BackendTLSPolicy should have one target ref") + g.Expect(string(policy.Spec.TargetRefs[0].Name)).To(Equal(solrCloud.CommonServiceName()), "BackendTLSPolicy should target common service") + g.Expect(string(policy.Spec.TargetRefs[0].Kind)).To(Equal("Service"), "BackendTLSPolicy should target Service kind") + g.Expect(string(policy.Spec.Validation.Hostname)).To(Equal(solrCloud.CommonServiceName()), "BackendTLSPolicy hostname should match service name") + g.Expect(policy.Spec.Validation.CACertificateRefs).To(HaveLen(1), "BackendTLSPolicy should have one CA cert ref") + g.Expect(string(policy.Spec.Validation.CACertificateRefs[0].Name)).To(Equal(caCertConfigMapName), "CA cert ref name should match") + g.Expect(string(policy.Spec.Validation.CACertificateRefs[0].Kind)).To(Equal("ConfigMap"), "CA cert ref kind should default to ConfigMap") + g.Expect(string(policy.Spec.Validation.CACertificateRefs[0].Group)).To(Equal(""), "CA cert ref group should be empty (core API)") + }) + Expect(commonPolicy).ToNot(BeNil()) + + By("verifying Node BackendTLSPolicy resources were created correctly") + for _, nodeName := range nodeNames { + nodePolicy := expectBackendTLSPolicyWithChecks(ctx, solrCloud, solrCloud.NodeBackendTLSPolicyName(nodeName), func(g Gomega, policy *gatewayv1.BackendTLSPolicy) { + g.Expect(policy.Spec.TargetRefs).To(HaveLen(1), "BackendTLSPolicy should have one target ref") + g.Expect(string(policy.Spec.TargetRefs[0].Name)).To(Equal(nodeName), "BackendTLSPolicy should target node service") + g.Expect(string(policy.Spec.Validation.Hostname)).To(Equal(nodeName), "BackendTLSPolicy hostname should match node service name") + g.Expect(policy.Spec.Validation.CACertificateRefs).To(HaveLen(1), "BackendTLSPolicy should have one CA cert ref") + }) + Expect(nodePolicy).ToNot(BeNil()) + } + + By("updating BackendTLSPolicy to use wellKnownCACertificates") + expectSolrCloudWithChecks(ctx, solrCloud, func(g Gomega, found *solrv1beta1.SolrCloud) { + wellKnown := "System" + found.Spec.SolrAddressability.External.Gateway.BackendTLSPolicy = &solrv1beta1.SolrBackendTLSPolicy{ + WellKnownCACertificates: &wellKnown, + } + g.Expect(k8sClient.Update(ctx, found)).To(Succeed(), "Couldn't update the solrCloud BackendTLSPolicy to use wellKnownCACertificates") + }) + + By("verifying BackendTLSPolicy was updated to use wellKnownCACertificates") + expectBackendTLSPolicyWithChecks(ctx, solrCloud, solrCloud.CommonBackendTLSPolicyName(), func(g Gomega, policy *gatewayv1.BackendTLSPolicy) { + g.Expect(policy.Spec.Validation.CACertificateRefs).To(BeNil(), "CACertificateRefs should be nil when using wellKnownCACertificates") + g.Expect(policy.Spec.Validation.WellKnownCACertificates).ToNot(BeNil(), "WellKnownCACertificates should be set") + g.Expect(string(*policy.Spec.Validation.WellKnownCACertificates)).To(Equal("System"), "WellKnownCACertificates should be 'System'") + }) + + By("disabling node external addressability and verifying node BackendTLSPolicy resources are deleted") + expectSolrCloudWithChecks(ctx, solrCloud, func(g Gomega, found *solrv1beta1.SolrCloud) { + found.Spec.SolrAddressability.External.HideNodes = true + g.Expect(k8sClient.Update(ctx, found)).To(Succeed(), "Couldn't update the solrCloud to hide nodes") + }) + + By("verifying node BackendTLSPolicy resources were deleted") + for _, nodeName := range nodeNames { + eventuallyExpectNoBackendTLSPolicy(ctx, solrCloud, solrCloud.NodeBackendTLSPolicyName(nodeName)) + } + + By("verifying common BackendTLSPolicy still exists") + expectBackendTLSPolicy(ctx, solrCloud, solrCloud.CommonBackendTLSPolicyName()) + + By("removing BackendTLSPolicy configuration") + expectSolrCloudWithChecks(ctx, solrCloud, func(g Gomega, found *solrv1beta1.SolrCloud) { + found.Spec.SolrAddressability.External.Gateway.BackendTLSPolicy = nil + g.Expect(k8sClient.Update(ctx, found)).To(Succeed(), "Couldn't update the solrCloud to remove BackendTLSPolicy") + }) + + By("verifying all BackendTLSPolicy resources were deleted") + eventuallyExpectNoBackendTLSPolicy(ctx, solrCloud, solrCloud.CommonBackendTLSPolicyName()) + }) + + It("Cleans up BackendTLSPolicy when changing from Gateway method", func(ctx context.Context) { + By("enabling BackendTLSPolicy") + expectSolrCloudWithChecks(ctx, solrCloud, func(g Gomega, found *solrv1beta1.SolrCloud) { + found.Spec.SolrAddressability.External.Gateway.BackendTLSPolicy = &solrv1beta1.SolrBackendTLSPolicy{ + CACertificateRefs: []solrv1beta1.GatewayCertificateReference{ + { + Name: caCertConfigMapName, + }, + }, + } + g.Expect(k8sClient.Update(ctx, found)).To(Succeed(), "Couldn't update the solrCloud to add BackendTLSPolicy") + }) + + By("verifying BackendTLSPolicy resources exist") + expectBackendTLSPolicy(ctx, solrCloud, solrCloud.CommonBackendTLSPolicyName()) + nodeNames := solrCloud.GetAllSolrPodNames() + for _, nodeName := range nodeNames { + expectBackendTLSPolicy(ctx, solrCloud, solrCloud.NodeBackendTLSPolicyName(nodeName)) + } + + By("changing external addressability method from Gateway to Ingress") + expectSolrCloudWithChecks(ctx, solrCloud, func(g Gomega, found *solrv1beta1.SolrCloud) { + found.Spec.SolrAddressability.External.Method = solrv1beta1.Ingress + found.Spec.SolrAddressability.External.Gateway = nil + g.Expect(k8sClient.Update(ctx, found)).To(Succeed(), "Couldn't update the solrCloud to change method to Ingress") + }) + + By("verifying all BackendTLSPolicy resources were cleaned up") + eventuallyExpectNoBackendTLSPolicy(ctx, solrCloud, solrCloud.CommonBackendTLSPolicyName()) + for _, nodeName := range nodeNames { + eventuallyExpectNoBackendTLSPolicy(ctx, solrCloud, solrCloud.NodeBackendTLSPolicyName(nodeName)) + } + + By("verifying HTTPRoutes were also cleaned up") + eventuallyExpectNoHTTPRoute(ctx, solrCloud, solrCloud.CommonHTTPRouteName()) + for _, nodeName := range nodeNames { + eventuallyExpectNoHTTPRoute(ctx, solrCloud, solrCloud.NodeHTTPRouteName(nodeName)) + } + }) + }) +}) diff --git a/tests/e2e/suite_test.go b/tests/e2e/suite_test.go index b63d2275..faa7959e 100644 --- a/tests/e2e/suite_test.go +++ b/tests/e2e/suite_test.go @@ -50,6 +50,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client/config" logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/log/zap" + gatewayv1 "sigs.k8s.io/gateway-api/apis/v1" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -109,6 +110,7 @@ var _ = SynchronizedBeforeSuite(func(ctx context.Context) { k8sConfig.Timeout = time.Minute Expect(zkApi.AddToScheme(scheme.Scheme)).To(Succeed()) Expect(certManagerApi.AddToScheme(scheme.Scheme)).To(Succeed()) + Expect(gatewayv1.AddToScheme(scheme.Scheme)).To(Succeed()) k8sClient, err = client.New(k8sConfig, client.Options{Scheme: scheme.Scheme}) Expect(err).NotTo(HaveOccurred(), "Could not create controllerRuntime Kubernetes client") @@ -154,6 +156,7 @@ var _ = SynchronizedBeforeSuite(func(ctx context.Context) { Expect(solrv1beta1.AddToScheme(scheme.Scheme)).To(Succeed()) Expect(zkApi.AddToScheme(scheme.Scheme)).To(Succeed()) Expect(certManagerApi.AddToScheme(scheme.Scheme)).To(Succeed()) + Expect(gatewayv1.AddToScheme(scheme.Scheme)).To(Succeed()) k8sClient, err = client.New(k8sConfig, client.Options{Scheme: scheme.Scheme}) Expect(err).NotTo(HaveOccurred(), "Could not create controllerRuntime Kubernetes client") diff --git a/tests/scripts/manage_e2e_tests.sh b/tests/scripts/manage_e2e_tests.sh index c9cfd7ff..51081224 100755 --- a/tests/scripts/manage_e2e_tests.sh +++ b/tests/scripts/manage_e2e_tests.sh @@ -83,7 +83,9 @@ if [[ "${SOLR_IMAGE}" != *":"* ]]; then fi IFS=$'\036'; RAW_GINKGO=(${RAW_GINKGO:-}); unset IFS -CLUSTER_NAME="$(echo "solr-op-e2e-${OPERATOR_IMAGE##*:}-k-${KUBERNETES_VERSION}-s-${SOLR_IMAGE##*:}" | tr '[:upper:]' '[:lower:]' | sed "s/snapshot/snap/" | sed "s/prerelease/pre/")" +CLUSTER_NAME_PREFIX="solr-op-e2e" +CLUSTER_NAME="$(echo "${CLUSTER_NAME_PREFIX}-${OPERATOR_IMAGE##*:}-k-${KUBERNETES_VERSION}-s-${SOLR_IMAGE##*:}" | tr '[:upper:]' '[:lower:]' | sed "s/snapshot/snap/" | sed "s/prerelease/pre/")" +export CLUSTER_NAME_PREFIX export CLUSTER_NAME export KUBE_CONTEXT="kind-${CLUSTER_NAME}" export KUBERNETES_VERSION @@ -99,6 +101,9 @@ export LEAVE_KIND_CLUSTER_ON_SUCCESS="${LEAVE_KIND_CLUSTER_ON_SUCCESS:-false}" # export RAWFILE_LOCAL_PV_VERSION=0.13.1 export CERT_MANAGER_VERSION=1.17.4 export CERT_MANAGER_CSI_DRIVER_VERSION=0.5.0 +# Keep in sync with the sigs.k8s.io/gateway-api version in go.mod. The standard channel is used, which includes +# the GA resources the operator manages (HTTPRoute and BackendTLSPolicy). +export GATEWAY_API_VERSION=1.6.1 function add_image_to_kind_repo_if_local() { IMAGE="$1" @@ -117,7 +122,28 @@ function add_image_to_kind_repo_if_local() { fi } -# These gingko params are customized via the following envVars +function start_cloud_provider_kind() { + # cloud-provider-kind is a host-level daemon (LoadBalancer + Gateway API support) shared by every KinD + # cluster on the host, so only one instance is needed. Skip if it is already running. + if pgrep -f "cloud-provider-kind" >/dev/null 2>&1; then + printf "cloud-provider-kind is already running.\n\n" + return + fi + printf "Starting cloud-provider-kind\n\n" + # nohup + disown so the daemon outlives this script invocation (e.g. the create-cluster action) + nohup cloud-provider-kind >"${TMPDIR:-/tmp}/cloud-provider-kind.log" 2>&1 & + disown +} + +function stop_cloud_provider_kind_if_unused() { + # The daemon is shared by every KinD cluster on the host, so only stop it once none of the integration-test + # clusters (named with the "solr-op-e2e" prefix) remain. This leaves it running for unrelated KinD clusters + # and for other concurrent test runs. + if ! kind get clusters 2>/dev/null | grep -q "^${CLUSTER_NAME_PREFIX}"; then + pkill -f "cloud-provider-kind" 2>/dev/null || true + fi +} + GINKGO_PARAM_NAMES=(--seed --procs --focus-file --label-filter --focus --skip) GINKGO_PARAM_ENVS=( TEST_SEED TEST_PARALLELISM TEST_FILES TEST_LABELS TEST_FILTER TEST_SKIP) function run_tests() { @@ -152,6 +178,7 @@ function export_kubeconfig() { function delete_cluster() { kind delete clusters "${CLUSTER_NAME}" + stop_cloud_provider_kind_if_unused } function start_cluster() { @@ -201,6 +228,15 @@ function setup_cluster() { helm upgrade -i -n cert-manager --create-namespace cert-manager cert-manager/cert-manager --version "${CERT_MANAGER_VERSION}" --set installCRDs=true helm upgrade -i -n cert-manager cert-manager-csi-driver cert-manager/cert-manager-csi-driver --version "${CERT_MANAGER_CSI_DRIVER_VERSION}" echo "" + + printf "Setup KinD Network Provider\n" + kubectl label node "${CLUSTER_NAME}-control-plane" node.kubernetes.io/exclude-from-external-load-balancers- + start_cloud_provider_kind + + printf "Installing Gateway API CRDs\n" + # Server-side apply is required: the Gateway API CRDs exceed the client-side apply annotation size limit. + kubectl apply --server-side -f "https://github.com/kubernetes-sigs/gateway-api/releases/download/v${GATEWAY_API_VERSION}/standard-install.yaml" + echo "" } case "$ACTION" in