diff --git a/api/v1alpha/instance_types.go b/api/v1alpha/instance_types.go index 9d2aa210..c6452cd3 100644 --- a/api/v1alpha/instance_types.go +++ b/api/v1alpha/instance_types.go @@ -813,6 +813,12 @@ type InstanceStatus struct { // Network interface information NetworkInterfaces []InstanceNetworkInterfaceStatus `json:"networkInterfaces,omitempty"` + // DNS reports assigned private names and publication readiness per network. + // Resolver access and record eligibility are managed by platform services. + // +kubebuilder:validation:Optional + // +listType=atomic + DNS []InstanceDNSStatus `json:"dns,omitempty"` + // Controller contains status information about the controller managing the instance. // // +kubebuilder:validation:Optional @@ -826,6 +832,18 @@ type InstanceStatus struct { Suspended bool `json:"suspended,omitempty"` } +// InstanceDNSStatus describes private name publication within one network. +type InstanceDNSStatus struct { + // Network identifies the attached network in the owning project. + Network networkingv1alpha.NetworkRef `json:"network"` + // Hostnames contains the fully qualified names allocated for this Instance. + Hostnames []string `json:"hostnames,omitempty"` + // Conditions reports DNS readiness independently of runtime readiness. + // +listType=map + // +listMapKey=type + Conditions []metav1.Condition `json:"conditions,omitempty"` +} + type InstanceControllerStatus struct { // ObservedTemplateHash is the hash of the instance template applied for this instance. // diff --git a/api/v1alpha/labels.go b/api/v1alpha/labels.go index 771883a6..a5bf8b09 100644 --- a/api/v1alpha/labels.go +++ b/api/v1alpha/labels.go @@ -9,6 +9,8 @@ const ( WorkloadDeploymentUIDLabel = LabelNamespace + "/workload-deployment-uid" InstanceIndexLabel = LabelNamespace + "/instance-index" + // InstanceSourceUIDLabel pins the edge Instance lifetime on its projections. + InstanceSourceUIDLabel = LabelNamespace + "/source-instance-uid" // WorkloadDeploymentNameLabel carries the name of the WorkloadDeployment // that owns an Instance. Stamped at creation and kept current on updates. diff --git a/api/v1alpha/zz_generated.deepcopy.go b/api/v1alpha/zz_generated.deepcopy.go index cae177b8..4d004faa 100644 --- a/api/v1alpha/zz_generated.deepcopy.go +++ b/api/v1alpha/zz_generated.deepcopy.go @@ -513,6 +513,34 @@ func (in *InstanceControllerStatus) DeepCopy() *InstanceControllerStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *InstanceDNSStatus) DeepCopyInto(out *InstanceDNSStatus) { + *out = *in + out.Network = in.Network + if in.Hostnames != nil { + in, out := &in.Hostnames, &out.Hostnames + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make([]metav1.Condition, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new InstanceDNSStatus. +func (in *InstanceDNSStatus) DeepCopy() *InstanceDNSStatus { + if in == nil { + return nil + } + out := new(InstanceDNSStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *InstanceList) DeepCopyInto(out *InstanceList) { *out = *in @@ -811,6 +839,13 @@ func (in *InstanceStatus) DeepCopyInto(out *InstanceStatus) { (*in)[i].DeepCopyInto(&(*out)[i]) } } + if in.DNS != nil { + in, out := &in.DNS, &out.DNS + *out = make([]InstanceDNSStatus, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } if in.Controller != nil { in, out := &in.Controller, &out.Controller *out = new(InstanceControllerStatus) diff --git a/cmd/main.go b/cmd/main.go index 8ea5e8b5..14341be0 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -159,7 +159,9 @@ func main() { flag.StringVar(&featureGatesFlag, "feature-gates", "", "A set of key=value pairs that describe feature gates for the compute operator. "+ "Example: --feature-gates=NetworkingIntegration=true. "+ - "Available features: NetworkingIntegration (default=false).") + "Available features: NetworkingIntegration, PodNetworkAttachment, RuntimeClasses, InstanceTypes, "+ + "InstanceConsoleSessions, "+ + "InternalDNSPublishing (all default=false).") opts := zap.Options{ Development: true, @@ -181,6 +183,7 @@ func main() { "RuntimeClasses", features.FeatureGate.Enabled(features.RuntimeClasses), "InstanceTypes", features.FeatureGate.Enabled(features.InstanceTypes), "InstanceConsoleSessions", features.FeatureGate.Enabled(features.InstanceConsoleSessions), + "InternalDNSPublishing", features.FeatureGate.Enabled(features.InternalDNSPublishing), ) ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts))) @@ -462,6 +465,39 @@ func main() { } } + // DNS publication is its own asynchronous management-plane controller. It is + // not registered at all while its default-off feature gate is disabled. + if features.FeatureGate.Enabled(features.InternalDNSPublishing) { + if serverConfig.Discovery.Mode != multiclusterproviders.ProviderMilo || !enableManagementControllers { + setupLog.Error(nil, + "InternalDNSPublishing requires Milo discovery mode and management controllers") + os.Exit(1) + } + identityConfig, identityErr := serverConfig.Discovery.DiscoveryRestConfig() + if identityErr != nil { + setupLog.Error(identityErr, "unable to load internal DNS project discovery credentials") + os.Exit(1) + } + identityClient, identityErr := client.New(identityConfig, client.Options{Scheme: scheme}) + if identityErr != nil { + setupLog.Error(identityErr, "unable to create internal DNS project discovery client") + os.Exit(1) + } + observers, observationErr := internalDNSObservationSources(serverConfig.InternalDNS) + if observationErr != nil { + setupLog.Error(observationErr, "invalid internal DNS network observation configuration") + os.Exit(1) + } + dnsPublisher := &controller.InternalDNSPublisherReconciler{ + ResolveProjectIdentity: controller.InternalDNSProjectIdentityResolver(identityClient), + ObservationSources: observers, + } + if err = dnsPublisher.SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "InternalDNSPublisher") + os.Exit(1) + } + } + // Instance types live in project control planes, which only the management // manager engages; cells read instance types but don't maintain them. if enableManagementControllers && features.FeatureGate.Enabled(features.InstanceTypes) { @@ -1029,3 +1065,31 @@ func instanceTypeCatalogSource( func computeWatchProviderClaims(mode multiclusterproviders.Provider) bool { return mode == multiclusterproviders.ProviderMilo } + +func internalDNSObservationSources( + cfg config.InternalDNSConfig, +) (map[string]controller.InternalDNSObservationSource, error) { + sources := make(map[string]controller.InternalDNSObservationSource, len(cfg.ObservationSources)) + for i, source := range cfg.ObservationSources { + if source.Location == "" || source.KubeconfigPath == "" { + return nil, fmt.Errorf("internalDNS.observationSources[%d] requires location and kubeconfigPath", i) + } + if _, exists := sources[source.Location]; exists { + return nil, fmt.Errorf("duplicate internal DNS observation location %q", source.Location) + } + cfg, err := clientcmd.BuildConfigFromFlags("", source.KubeconfigPath) + if err != nil { + return nil, fmt.Errorf("load network observation credentials for %q: %w", source.Location, err) + } + cfg.Timeout = 5 * time.Second + reader, err := client.New(cfg, client.Options{Scheme: scheme}) + if err != nil { + return nil, fmt.Errorf("create network observation client for %q: %w", source.Location, err) + } + sources[source.Location] = controller.InternalDNSObservationSource{Reader: reader} + } + if len(sources) == 0 { + return nil, errors.New("internalDNS.observationSources must contain at least one authenticated edge API") + } + return sources, nil +} diff --git a/cmd/main_test.go b/cmd/main_test.go index df27d639..aff5b53d 100644 --- a/cmd/main_test.go +++ b/cmd/main_test.go @@ -19,9 +19,42 @@ import ( locationsv1alpha1 "go.miloapis.com/locations/api/v1alpha1" multiclusterproviders "go.miloapis.com/milo/pkg/multicluster-runtime" + "go.datum.net/compute/internal/config" "go.datum.net/compute/internal/locations" ) +func TestInternalDNSObservationSourcesRequireExplicitCredentials(t *testing.T) { + _, err := internalDNSObservationSources(config.InternalDNSConfig{}) + require.Error(t, err) + _, err = internalDNSObservationSources(config.InternalDNSConfig{ + ObservationSources: []config.InternalDNSObservationSource{{Location: "dfw"}}, + }) + require.Error(t, err) + path := filepath.Join(t.TempDir(), "edge.kubeconfig") + require.NoError(t, os.WriteFile(path, []byte(`apiVersion: v1 +kind: Config +clusters: + - name: edge + cluster: + server: https://127.0.0.1:6443 +contexts: + - name: edge + context: + cluster: edge +current-context: edge +`), 0o600)) + source := config.InternalDNSObservationSource{Location: "dfw", KubeconfigPath: path} + sources, err := internalDNSObservationSources(config.InternalDNSConfig{ + ObservationSources: []config.InternalDNSObservationSource{source}, + }) + require.NoError(t, err) + require.NotNil(t, sources["dfw"].Reader) + _, err = internalDNSObservationSources(config.InternalDNSConfig{ + ObservationSources: []config.InternalDNSObservationSource{source, source}, + }) + require.Error(t, err) +} + // TestComputeWatchProviderClaims is the #171 guard: quota enforcement (and thus // the ResourceClaim watch) is wired only in Milo mode. Single/cluster mode must // stay false, so the manager never engages a ResourceClaim watch against a cell diff --git a/config/base/crd/bases/compute.datumapis.com_instances.yaml b/config/base/crd/bases/compute.datumapis.com_instances.yaml index a7336baf..c01dc749 100644 --- a/config/base/crd/bases/compute.datumapis.com_instances.yaml +++ b/config/base/crd/bases/compute.datumapis.com_instances.yaml @@ -1260,6 +1260,103 @@ spec: required: - observedTemplateHash type: object + dns: + description: |- + DNS reports assigned private names and publication readiness per network. + Resolver access and record eligibility are managed by platform services. + items: + description: InstanceDNSStatus describes private name publication + within one network. + properties: + conditions: + description: Conditions reports DNS readiness independently + of runtime readiness. + items: + description: Condition contains details for one aspect of + the current state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, + Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + hostnames: + description: Hostnames contains the fully qualified names allocated + for this Instance. + items: + type: string + type: array + network: + description: Network identifies the attached network in the + owning project. + properties: + name: + description: The network name + type: string + namespace: + description: |- + The network namespace. + + Defaults to the namespace for the type the reference is embedded in. + type: string + required: + - name + type: object + required: + - network + type: object + type: array + x-kubernetes-list-type: atomic networkInterfaces: description: Network interface information items: diff --git a/config/components/internal-dns-rbac/kustomization.yaml b/config/components/internal-dns-rbac/kustomization.yaml new file mode 100644 index 00000000..c451c7b9 --- /dev/null +++ b/config/components/internal-dns-rbac/kustomization.yaml @@ -0,0 +1,5 @@ +apiVersion: kustomize.config.k8s.io/v1alpha1 +kind: Component +resources: + - role.yaml + - role_binding.yaml diff --git a/config/components/internal-dns-rbac/role.yaml b/config/components/internal-dns-rbac/role.yaml new file mode 100644 index 00000000..38fc1ec2 --- /dev/null +++ b/config/components/internal-dns-rbac/role.yaml @@ -0,0 +1,17 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: compute-internal-dns-publisher +rules: + - apiGroups: ["dns.networking.miloapis.com"] + resources: ["dnsresolvercontexts"] + verbs: ["get", "list"] + - apiGroups: ["dns.networking.miloapis.com"] + resources: ["dnsregistrations", "dnsrecordcontributions"] + verbs: ["get", "list", "create", "update", "patch", "delete"] + - apiGroups: ["dns.networking.miloapis.com"] + resources: ["dnsrecordcontributions/status"] + verbs: ["get", "update", "patch"] + - apiGroups: ["compute.datumapis.com"] + resources: ["instances/status"] + verbs: ["get", "patch"] diff --git a/config/components/internal-dns-rbac/role_binding.yaml b/config/components/internal-dns-rbac/role_binding.yaml new file mode 100644 index 00000000..dc6b0a43 --- /dev/null +++ b/config/components/internal-dns-rbac/role_binding.yaml @@ -0,0 +1,12 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: compute-internal-dns-publisher +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: compute-internal-dns-publisher +subjects: + - kind: ServiceAccount + name: compute-manager + namespace: compute-system diff --git a/docs/enhancements/internal-dns-instance-publishing.md b/docs/enhancements/internal-dns-instance-publishing.md new file mode 100644 index 00000000..b450bb99 --- /dev/null +++ b/docs/enhancements/internal-dns-instance-publishing.md @@ -0,0 +1,59 @@ +# Internal DNS instance publishing + +Compute publishes private instance names in each attached network's managed +zone. Network services requests `datum.internal` by default. Compute reads the +allocated suffix, so users attach workloads to networks without selecting zones. +An instance named `web-01` receives `web-01.datum.internal`. Overlapping domains +remain isolated by the network's DNS context. + +`Instance.status.dns` reports hostnames and publication readiness per network. +DNS failures do not affect runtime readiness. Application readiness does not +remove an instance identity name. + +```yaml +status: + dns: + - network: + name: application + namespace: production + hostnames: [web-01.datum.internal] + conditions: + - type: Ready + status: "True" + reason: Published + message: Private name is published +``` + +Enable the default-off `InternalDNSPublishing` feature gate after provisioning +DNS, project permissions, and read-only edge observation credentials. Compute +resolves project identities through Milo discovery, including newly created projects. + +```yaml +apiVersion: apiserver.config.datumapis.com/v1alpha1 +kind: WorkloadOperator +internalDNS: + observationSources: + - location: dfw + # Authenticate to the edge API that owns instance observations. + kubeconfigPath: /etc/compute/edge-dfw.kubeconfig +``` + +Compute selects the network's resolver context and reads its managed zone. It +reserves a name through `DNSRegistration` and publishes addresses and eligibility +atomically through `DNSRecordContribution`. Project RBAC authorizes writes. +Object lifetime references and contribution generations prevent stale publishers +from changing replacement resources. + +Only allocated, programmed private host addresses are eligible. Confirmed +interface withdrawal, stopped runtime, or unhealthy node withdraws addresses. +An unavailable observation source or unknown node health leaves published records +unchanged and reports unknown observation health. Records have no renewal +requirement. Instance deletion reclaims records through project owner references. + +Observation credentials read Instances, NetworkInterfaces, NetworkInterfaceClaims, +NetworkContexts, Pods, Nodes, and project namespaces. Scope namespaced reads to +supported edge namespaces. These credentials cannot publish DNS records. + +Workload providers apply the network's resolver address and allocated search +domain to guests. Resolver delivery belongs to network services. The publisher +contract does not expose access deadlines on public NetworkInterfaces. diff --git a/go.mod b/go.mod index 3e55735f..e127580b 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module go.datum.net/compute go 1.26.4 require ( + github.com/evanphx/json-patch/v5 v5.9.11 github.com/KimMachineGun/automemlimit v0.7.5 github.com/google/go-cmp v0.7.0 github.com/karmada-io/api v1.15.0 @@ -121,7 +122,6 @@ require ( github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/emicklei/go-restful/v3 v3.13.0 // indirect - github.com/evanphx/json-patch/v5 v5.9.11 // indirect github.com/felixge/httpsnoop v1.1.0 // indirect github.com/fsnotify/fsnotify v1.10.1 // indirect github.com/fxamacker/cbor/v2 v2.9.0 // indirect diff --git a/internal/config/config.go b/internal/config/config.go index 2a7540d0..f5411d0f 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -44,6 +44,11 @@ type WorkloadOperator struct { // ReferencedData configures the ReferencedDataController. ReferencedData ReferencedDataConfig `json:"referencedData,omitempty"` + // InternalDNS configures project identities for the optional instance DNS + // publisher. It is ignored unless the InternalDNSPublishing feature gate is + // enabled. + InternalDNS InternalDNSConfig `json:"internalDNS,omitempty"` + // LocationSource names the API group locations are read from. Use // "NetworkServices" for networking.datumapis.com LocationBindings and // ServingLocations, or "Locations" for the dedicated locations.miloapis.com @@ -52,6 +57,19 @@ type WorkloadOperator struct { LocationSource locations.Source `json:"locationSource,omitempty"` } +// +k8s:deepcopy-gen=true +type InternalDNSConfig struct { + // ObservationSources selects authenticated edge APIs for live interface and + // node observations. Credentials must have read-only observation permissions. + ObservationSources []InternalDNSObservationSource `json:"observationSources,omitempty"` +} + +// +k8s:deepcopy-gen=true +type InternalDNSObservationSource struct { + Location string `json:"location"` + KubeconfigPath string `json:"kubeconfigPath"` +} + func SetDefaults_WorkloadOperator(obj *WorkloadOperator) { if obj.LocationSource == "" { obj.LocationSource = locations.SourceNetworkServices diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 61c4ca33..98957865 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -28,6 +28,24 @@ func decode(t *testing.T, data string) *WorkloadOperator { return &cfg } +func TestInternalDNSConfigDecodeAndDeepCopy(t *testing.T) { + cfg := decode(t, ` +apiVersion: apiserver.config.datumapis.com/v1alpha1 +kind: WorkloadOperator +metricsServer: + bindAddress: "0" +internalDNS: + observationSources: + - location: dfw + kubeconfigPath: /etc/compute/edge.kubeconfig +`) + copy := cfg.DeepCopy() + copy.InternalDNS.ObservationSources[0].Location = "changed" + if cfg.InternalDNS.ObservationSources[0].Location != "dfw" { + t.Fatal("WorkloadOperator DeepCopy aliases observation identities") + } +} + func TestWebhookServer_Absent(t *testing.T) { cfg := decode(t, ` apiVersion: apiserver.config.datumapis.com/v1alpha1 diff --git a/internal/config/zz_generated.deepcopy.go b/internal/config/zz_generated.deepcopy.go index 191a9cbc..62fc8f11 100644 --- a/internal/config/zz_generated.deepcopy.go +++ b/internal/config/zz_generated.deepcopy.go @@ -66,6 +66,41 @@ func (in *FeatureFlagsConfig) DeepCopy() *FeatureFlagsConfig { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *InternalDNSConfig) DeepCopyInto(out *InternalDNSConfig) { + *out = *in + if in.ObservationSources != nil { + in, out := &in.ObservationSources, &out.ObservationSources + *out = make([]InternalDNSObservationSource, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new InternalDNSConfig. +func (in *InternalDNSConfig) DeepCopy() *InternalDNSConfig { + if in == nil { + return nil + } + out := new(InternalDNSConfig) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *InternalDNSObservationSource) DeepCopyInto(out *InternalDNSObservationSource) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new InternalDNSObservationSource. +func (in *InternalDNSObservationSource) DeepCopy() *InternalDNSObservationSource { + if in == nil { + return nil + } + out := new(InternalDNSObservationSource) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *MetricsServerConfig) DeepCopyInto(out *MetricsServerConfig) { *out = *in @@ -156,6 +191,7 @@ func (in *WorkloadOperator) DeepCopyInto(out *WorkloadOperator) { in.Discovery.DeepCopyInto(&out.Discovery) out.FeatureFlags = in.FeatureFlags in.ReferencedData.DeepCopyInto(&out.ReferencedData) + in.InternalDNS.DeepCopyInto(&out.InternalDNS) } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new WorkloadOperator. diff --git a/internal/controller/instance_controller.go b/internal/controller/instance_controller.go index 5ca6c907..5926b49a 100644 --- a/internal/controller/instance_controller.go +++ b/internal/controller/instance_controller.go @@ -1305,6 +1305,7 @@ func (r *InstanceReconciler) writeBackToUpstream(ctx context.Context, instance * computev1alpha.WorkloadUIDLabel: instance.Labels[computev1alpha.WorkloadUIDLabel], computev1alpha.WorkloadDeploymentUIDLabel: instance.Labels[computev1alpha.WorkloadDeploymentUIDLabel], computev1alpha.InstanceIndexLabel: instance.Labels[computev1alpha.InstanceIndexLabel], + computev1alpha.InstanceSourceUIDLabel: string(instance.UID), computev1alpha.WorkloadDeploymentNameLabel: instance.Labels[computev1alpha.WorkloadDeploymentNameLabel], computev1alpha.LocationLabel: instance.Labels[computev1alpha.LocationLabel], computev1alpha.WorkloadNameLabel: instance.Labels[computev1alpha.WorkloadNameLabel], diff --git a/internal/controller/instance_projector.go b/internal/controller/instance_projector.go index 071160f7..a84e7556 100644 --- a/internal/controller/instance_projector.go +++ b/internal/controller/instance_projector.go @@ -5,6 +5,7 @@ package controller import ( "context" "fmt" + "time" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -36,10 +37,9 @@ import ( // so that it is garbage-collected via cascading deletion when the deployment is // removed from the project cluster. // -// The projector never deletes anything. Reclamation belongs to hub ownership: -// hub Instances are owned by their hub WorkloadDeployment, and write-back is -// owner-gated, so an object the projector cannot resolve is not something for it -// to clean up. +// A changed source Instance UID replaces the projection so project-owned +// resources cannot survive into another instance lifetime. Deployment deletion +// reclaims projections through their owner references. // // The controller is registered on the leader-elected local manager so only the // elected replica writes projections, and watches Instances through the @@ -150,6 +150,18 @@ func (r *InstanceProjector) Reconcile(ctx context.Context, req ctrl.Request) (ct }, } + if err := projectCluster.GetAPIReader().Get(ctx, client.ObjectKeyFromObject(projection), projection); err != nil && !apierrors.IsNotFound(err) { + return ctrl.Result{}, err + } + oldSource := projection.Labels[computev1alpha.InstanceSourceUIDLabel] + newSource := downstreamInstance.Labels[computev1alpha.InstanceSourceUIDLabel] + if oldSource != "" && newSource != "" && oldSource != newSource { + if err := projectClient.Delete(ctx, projection, client.Preconditions{UID: &projection.UID, ResourceVersion: &projection.ResourceVersion}); client.IgnoreNotFound(err) != nil { + return ctrl.Result{}, err + } + return ctrl.Result{RequeueAfter: time.Millisecond}, nil + } + operationResult, err := controllerutil.CreateOrUpdate(ctx, projectClient, projection, func() error { // Propagate upstream tracking labels so consumers can filter by origin. if projection.Labels == nil { @@ -173,7 +185,10 @@ func (r *InstanceProjector) Reconcile(ctx context.Context, req ctrl.Request) (ct logger.Info("reconciled Instance projection", "operation", operationResult, "namespace", targetNamespace, "cluster", clusterName) // Status is a separate subresource. + dnsStatus := projection.Status.DNS projection.Status = downstreamInstance.Status + // Private name publication belongs to the project-side DNS publisher. + projection.Status.DNS = dnsStatus if err := projectClient.Status().Update(ctx, projection); err != nil && !apierrors.IsNotFound(err) { return ctrl.Result{}, fmt.Errorf("failed updating Instance projection status: %w", err) } diff --git a/internal/controller/instance_projector_test.go b/internal/controller/instance_projector_test.go index 729a9de5..2d3a9eda 100644 --- a/internal/controller/instance_projector_test.go +++ b/internal/controller/instance_projector_test.go @@ -33,6 +33,7 @@ import ( metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" computev1alpha "go.datum.net/compute/api/v1alpha" + networkingv1alpha "go.datum.net/network-services-operator/api/v1alpha" locationsv1alpha1 "go.miloapis.com/locations/api/v1alpha1" "go.miloapis.com/milo/pkg/downstreamclient" ) @@ -438,6 +439,35 @@ func TestInstanceProjector_SpecCopied(t *testing.T) { assert.Equal(t, "test-gate", projection.Spec.Controller.SchedulingGates[0].Name) } +func TestInstanceProjectorPreservesProjectDNSStatus(t *testing.T) { + upstream := projTestKarmadaInstance(map[string]string{computev1alpha.InstanceSourceUIDLabel: internalDNSTestEdgeUID}) + upstream.Status.DNS = []computev1alpha.InstanceDNSStatus{{Network: networkingv1alpha.NetworkRef{Name: "untrusted-edge-network", Namespace: "edge"}}} + network := networkingv1alpha.NetworkRef{Name: "project-network", Namespace: projTestProjNS} + projection := &computev1alpha.Instance{ObjectMeta: metav1.ObjectMeta{Name: projTestInstanceName, Namespace: projTestProjNS, Labels: map[string]string{computev1alpha.InstanceSourceUIDLabel: internalDNSTestEdgeUID}}, Status: computev1alpha.InstanceStatus{DNS: []computev1alpha.InstanceDNSStatus{{Network: network, Hostnames: []string{"web.datum.internal"}}}}} + cl := fake.NewClientBuilder().WithScheme(newProjectScheme()).WithObjects(projTestWorkloadDeployment(), projection).WithStatusSubresource(&computev1alpha.Instance{}).Build() + r := newTestProjector(newKarmadaFakeClient(upstream), cl) + _, err := r.Reconcile(context.Background(), projectorRequest()) + require.NoError(t, err) + require.NoError(t, cl.Get(context.Background(), client.ObjectKeyFromObject(projection), projection)) + require.Equal(t, network, projection.Status.DNS[0].Network) +} + +func TestInstanceProjectorReplacesDifferentSourceLifetime(t *testing.T) { + upstream := projTestKarmadaInstance(map[string]string{computev1alpha.InstanceSourceUIDLabel: "new-edge-instance"}) + projection := &computev1alpha.Instance{ObjectMeta: metav1.ObjectMeta{Name: projTestInstanceName, Namespace: projTestProjNS, UID: "old-project-instance", Labels: map[string]string{computev1alpha.InstanceSourceUIDLabel: "old-edge-instance"}}} + cl := fake.NewClientBuilder().WithScheme(newProjectScheme()).WithObjects(projTestWorkloadDeployment(), projection).WithStatusSubresource(&computev1alpha.Instance{}).Build() + r := newTestProjector(newKarmadaFakeClient(upstream), cl) + result, err := r.Reconcile(context.Background(), projectorRequest()) + require.NoError(t, err) + require.Positive(t, result.RequeueAfter) + require.True(t, apierrors.IsNotFound(cl.Get(context.Background(), client.ObjectKeyFromObject(projection), projection))) + _, err = r.Reconcile(context.Background(), projectorRequest()) + require.NoError(t, err) + require.NoError(t, cl.Get(context.Background(), client.ObjectKeyFromObject(projection), projection)) + require.Equal(t, "new-edge-instance", projection.Labels[computev1alpha.InstanceSourceUIDLabel]) + require.Empty(t, projection.Status.DNS) +} + // TestInstanceProjector_NamespaceResolution verifies that the projector resolves // the target project namespace directly from the UpstreamOwnerNamespaceLabel on // the Karmada Instance, landing the projection in the correct namespace. diff --git a/internal/controller/internaldns_observation.go b/internal/controller/internaldns_observation.go new file mode 100644 index 00000000..a21bcfb1 --- /dev/null +++ b/internal/controller/internaldns_observation.go @@ -0,0 +1,185 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package controller + +import ( + "context" + "fmt" + + computev1alpha "go.datum.net/compute/api/v1alpha" + networkingv1alpha "go.datum.net/network-services-operator/api/v1alpha" + corev1 "k8s.io/api/core/v1" + apimeta "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// InternalDNSObservedInterface reports confirmed interface eligibility. +type InternalDNSObservedInterface struct { + Addresses []networkingv1alpha.NetworkInterfaceAddress + Eligible bool +} + +type InternalDNSObserveFunc func(context.Context, client.Reader, string, *computev1alpha.Instance, computev1alpha.InstanceNetworkInterface) (InternalDNSObservedInterface, error) + +// InternalDNSObservationSource supplies an authenticated edge API reader. +type InternalDNSObservationSource struct { + Reader client.Reader +} + +// ObserveInternalDNSInterface checks interface programming and runtime health. +func ObserveInternalDNSInterface(ctx context.Context, project client.Reader, namespace string, instance *computev1alpha.Instance, nic computev1alpha.InstanceNetworkInterface, source InternalDNSObservationSource) (InternalDNSObservedInterface, error) { + var result InternalDNSObservedInterface + if nic.Network.Namespace != "" && nic.Network.Namespace != namespace { + return result, fmt.Errorf("interface observations require a network in the Instance namespace") + } + if source.Reader == nil { + return result, fmt.Errorf("no authenticated network observation source is configured for location %q", instance.Labels[computev1alpha.LocationLabel]) + } + var projectNS corev1.Namespace + if err := project.Get(ctx, client.ObjectKey{Name: namespace}, &projectNS); err != nil { + return result, err + } + if projectNS.UID == "" { + return result, fmt.Errorf("project namespace has no lifetime identity") + } + var network networkingv1alpha.Network + if err := project.Get(ctx, client.ObjectKey{Namespace: namespace, Name: nic.Network.Name}, &network); err != nil { + return result, err + } + if network.UID == "" || !network.DeletionTimestamp.IsZero() { + return result, nil + } + sourceNamespace := "ns-" + string(projectNS.UID) + var current computev1alpha.Instance + if err := source.Reader.Get(ctx, client.ObjectKey{Namespace: sourceNamespace, Name: instance.Name}, ¤t); err != nil { + return result, err + } + if current.UID == "" || string(current.UID) != instance.Labels[computev1alpha.InstanceSourceUIDLabel] || !current.DeletionTimestamp.IsZero() { + return result, fmt.Errorf("source Instance lifetime does not match the project projection") + } + iface, err := observeProgrammedInterface(ctx, source.Reader, ¤t, nic) + if err != nil || iface == nil { + return result, err + } + if err := verifyObservedNetwork(ctx, source.Reader, iface, &network, instance.Labels[computev1alpha.LocationLabel]); err != nil { + return result, err + } + nodeName, err := observeRuntimeNode(ctx, source.Reader, ¤t) + if err != nil || nodeName == "" { + return result, err + } + healthy, err := observeNodeHealth(ctx, source.Reader, nodeName) + if err != nil || !healthy { + return result, err + } + result.Addresses = append([]networkingv1alpha.NetworkInterfaceAddress(nil), iface.Spec.Addresses...) + result.Eligible = true + return result, nil +} + +func verifyObservedNetwork(ctx context.Context, reader client.Reader, iface *networkingv1alpha.NetworkInterface, network *networkingv1alpha.Network, location string) error { + if iface.Status.NetworkContextRef == nil || location == "" { + return fmt.Errorf("interface has no verified network context") + } + var networkContext networkingv1alpha.NetworkContext + if err := reader.Get(ctx, client.ObjectKey{Namespace: iface.Namespace, Name: iface.Status.NetworkContextRef.Name}, &networkContext); err != nil { + return err + } + if networkContext.UID == "" || !networkContext.DeletionTimestamp.IsZero() || networkContext.Labels[networkingv1alpha.NetworkUIDLabel] != string(network.UID) || networkContext.Spec.Network.Name != network.Name || networkContext.Spec.Location.Name != location { + return fmt.Errorf("interface network lifetime or location does not match the project") + } + return nil +} + +func observeProgrammedInterface(ctx context.Context, reader client.Reader, current *computev1alpha.Instance, nic computev1alpha.InstanceNetworkInterface) (*networkingv1alpha.NetworkInterface, error) { + var ifaceName string + for _, status := range current.Status.NetworkInterfaces { + if status.Name == internalDNSInterfaceName(nic) && status.NetworkInterfaceRef != nil { + ifaceName = status.NetworkInterfaceRef.Name + } + } + if ifaceName == "" { + return nil, nil + } + var iface networkingv1alpha.NetworkInterface + if err := reader.Get(ctx, client.ObjectKey{Namespace: current.Namespace, Name: ifaceName}, &iface); err != nil { + return nil, err + } + if iface.UID == "" || !iface.DeletionTimestamp.IsZero() || iface.Spec.Network.Name != nic.Network.Name || iface.Spec.InterfaceName != internalDNSInterfaceName(nic) || iface.Spec.ClaimRef == nil { + return nil, nil + } + unknown := false + for _, kind := range []string{"Allocated", "Programmed"} { + condition := apimeta.FindStatusCondition(iface.Status.Conditions, kind) + if condition == nil || condition.ObservedGeneration != iface.Generation || condition.Status == metav1.ConditionUnknown { + unknown = true + continue + } + if condition.Status == metav1.ConditionFalse { + return nil, nil + } + } + if unknown { + return nil, fmt.Errorf("interface %q programming health is unknown", iface.Name) + } + if iface.Spec.ClaimRef.Name != networkInterfaceClaimName(current.Name, internalDNSInterfaceName(nic)) { + return nil, fmt.Errorf("interface is held by a different instance slot") + } + var claim networkingv1alpha.NetworkInterfaceClaim + if err := reader.Get(ctx, client.ObjectKey{Namespace: current.Namespace, Name: iface.Spec.ClaimRef.Name}, &claim); err != nil { + return nil, err + } + if claim.UID == "" || !claim.DeletionTimestamp.IsZero() || claim.Spec.Network.Name != iface.Spec.Network.Name || claim.Status.NetworkInterfaceRef == nil || claim.Status.NetworkInterfaceRef.Name != iface.Name { + return nil, nil + } + return &iface, nil +} + +func observeRuntimeNode(ctx context.Context, reader client.Reader, current *computev1alpha.Instance) (string, error) { + var pods corev1.PodList + options := []client.ListOption{client.InNamespace(current.Namespace)} + deployment, index := current.Labels[computev1alpha.WorkloadDeploymentUIDLabel], current.Labels[computev1alpha.InstanceIndexLabel] + if deployment != "" && index != "" { + options = append(options, client.MatchingLabels{ + computev1alpha.WorkloadDeploymentUIDLabel: deployment, + computev1alpha.InstanceIndexLabel: index, + }) + } + if err := reader.List(ctx, &pods, options...); err != nil { + return "", err + } + var nodeName string + for i := range pods.Items { + pod := &pods.Items[i] + if metav1.IsControlledBy(pod, current) && pod.DeletionTimestamp.IsZero() && pod.Status.Phase == corev1.PodRunning && pod.Spec.NodeName != "" { + if nodeName != "" { + return "", fmt.Errorf("multiple running runtime Pods own this Instance") + } + nodeName = pod.Spec.NodeName + } + } + return nodeName, nil +} + +func observeNodeHealth(ctx context.Context, reader client.Reader, nodeName string) (bool, error) { + var node corev1.Node + if err := reader.Get(ctx, client.ObjectKey{Name: nodeName}, &node); err != nil { + return false, err + } + if node.UID == "" || !node.DeletionTimestamp.IsZero() { + return false, nil + } + for _, condition := range node.Status.Conditions { + if condition.Type != corev1.NodeReady { + continue + } + switch condition.Status { + case corev1.ConditionTrue: + return true, nil + case corev1.ConditionFalse: + return false, nil + } + } + return false, fmt.Errorf("node %q health is unknown", nodeName) +} diff --git a/internal/controller/internaldns_observation_test.go b/internal/controller/internaldns_observation_test.go new file mode 100644 index 00000000..e8708602 --- /dev/null +++ b/internal/controller/internaldns_observation_test.go @@ -0,0 +1,99 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package controller + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" + computev1alpha "go.datum.net/compute/api/v1alpha" + networkingv1alpha "go.datum.net/network-services-operator/api/v1alpha" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client/fake" +) + +func TestObserveInternalDNSInterface(t *testing.T) { + for _, tc := range []struct { + name string + mutate func(*computev1alpha.Instance, *networkingv1alpha.NetworkInterface, *corev1.Pod, *corev1.Node) + eligible bool + wantErr bool + networkUID types.UID + }{ + {name: "live network with unready application", eligible: true}, + {name: "replaced network lifetime", networkUID: "replacement-network", wantErr: true}, + {name: "different instance lifetime", mutate: func(i *computev1alpha.Instance, _ *networkingv1alpha.NetworkInterface, _ *corev1.Pod, _ *corev1.Node) { + i.UID = "replacement" + }, wantErr: true}, + {name: "stale programming generation", mutate: func(_ *computev1alpha.Instance, i *networkingv1alpha.NetworkInterface, _ *corev1.Pod, _ *corev1.Node) { + i.Generation++ + }, wantErr: true}, + {name: "unknown interface programming", mutate: func(_ *computev1alpha.Instance, i *networkingv1alpha.NetworkInterface, _ *corev1.Pod, _ *corev1.Node) { + i.Status.Conditions[1].Status = metav1.ConditionUnknown + }, wantErr: true}, + {name: "different network", mutate: func(_ *computev1alpha.Instance, i *networkingv1alpha.NetworkInterface, _ *corev1.Pod, _ *corev1.Node) { + i.Spec.Network.Name = internalDNSTestForeignValue + }}, + {name: "different claim", mutate: func(_ *computev1alpha.Instance, i *networkingv1alpha.NetworkInterface, _ *corev1.Pod, _ *corev1.Node) { + i.Spec.ClaimRef.Name = internalDNSTestForeignValue + }, wantErr: true}, + {name: "runtime exited", mutate: func(_ *computev1alpha.Instance, _ *networkingv1alpha.NetworkInterface, p *corev1.Pod, _ *corev1.Node) { + p.Status.Phase = corev1.PodFailed + }}, + {name: "different runtime slot", mutate: func(_ *computev1alpha.Instance, _ *networkingv1alpha.NetworkInterface, p *corev1.Pod, _ *corev1.Node) { + p.Labels[computev1alpha.InstanceIndexLabel] = "different-slot" + }}, + {name: "node unavailable", mutate: func(_ *computev1alpha.Instance, _ *networkingv1alpha.NetworkInterface, _ *corev1.Pod, n *corev1.Node) { + n.Status.Conditions[0].Status = corev1.ConditionFalse + }}, + {name: "unknown node health", mutate: func(_ *computev1alpha.Instance, _ *networkingv1alpha.NetworkInterface, _ *corev1.Pod, n *corev1.Node) { + n.Status.Conditions[0].Status = corev1.ConditionUnknown + }, wantErr: true}, + } { + t.Run(tc.name, func(t *testing.T) { + scheme := runtime.NewScheme() + for _, add := range []func(*runtime.Scheme) error{corev1.AddToScheme, computev1alpha.AddToScheme, networkingv1alpha.AddToScheme} { + require.NoError(t, add(scheme)) + } + instance := &computev1alpha.Instance{ObjectMeta: metav1.ObjectMeta{Name: internalDNSTestInstanceName, Namespace: internalDNSTestObservationNamespace, UID: "project-instance", Labels: map[string]string{computev1alpha.InstanceSourceUIDLabel: internalDNSTestEdgeUID, computev1alpha.LocationLabel: "dfw"}}, Spec: computev1alpha.InstanceSpec{NetworkInterfaces: []computev1alpha.InstanceNetworkInterface{{Network: networkingv1alpha.NetworkRef{Name: internalDNSTestNetworkName}}}}} + edgeInstance := &computev1alpha.Instance{ObjectMeta: metav1.ObjectMeta{Name: instance.Name, Namespace: "ns-project-namespace", UID: internalDNSTestEdgeUID}, Status: computev1alpha.InstanceStatus{NetworkInterfaces: []computev1alpha.InstanceNetworkInterfaceStatus{{Name: defaultInterfaceName, NetworkInterfaceRef: &networkingv1alpha.LocalNetworkInterfaceRef{Name: "interface"}}}}} + iface := &networkingv1alpha.NetworkInterface{ObjectMeta: metav1.ObjectMeta{Name: "interface", Namespace: edgeInstance.Namespace, UID: "interface-uid", Generation: 2}, Spec: networkingv1alpha.NetworkInterfaceSpec{Network: networkingv1alpha.LocalNetworkRef{Name: internalDNSTestNetworkName}, InterfaceName: defaultInterfaceName, ClaimRef: &networkingv1alpha.NetworkInterfaceClaimRef{Name: networkInterfaceClaimName(internalDNSTestInstanceName, defaultInterfaceName)}, Addresses: []networkingv1alpha.NetworkInterfaceAddress{{Address: "10.20.0.10/32"}}}, Status: networkingv1alpha.NetworkInterfaceStatus{Conditions: []metav1.Condition{{Type: "Allocated", Status: metav1.ConditionTrue, ObservedGeneration: 2}, {Type: "Programmed", Status: metav1.ConditionTrue, ObservedGeneration: 2}}}} + pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "runtime", Namespace: edgeInstance.Namespace, OwnerReferences: []metav1.OwnerReference{{APIVersion: computev1alpha.GroupVersion.String(), Kind: internalDNSInstanceKind, Name: edgeInstance.Name, UID: edgeInstance.UID, Controller: ptrBool(true)}}}, Spec: corev1.PodSpec{NodeName: "worker"}, Status: corev1.PodStatus{Phase: corev1.PodRunning, Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: corev1.ConditionFalse}}}} + node := &corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: "worker", UID: "node-uid"}, Status: corev1.NodeStatus{Conditions: []corev1.NodeCondition{{Type: corev1.NodeReady, Status: corev1.ConditionTrue}}}} + edgeInstance.Labels = map[string]string{computev1alpha.WorkloadDeploymentUIDLabel: "dns-deployment", computev1alpha.InstanceIndexLabel: "0"} + pod.Labels = map[string]string{computev1alpha.WorkloadDeploymentUIDLabel: "dns-deployment", computev1alpha.InstanceIndexLabel: "0"} + if tc.mutate != nil { + tc.mutate(edgeInstance, iface, pod, node) + } + iface.Status.NetworkContextRef = &networkingv1alpha.LocalNetworkContextRef{Name: "context"} + networkUID := types.UID("network-uid") + if tc.networkUID != "" { + networkUID = tc.networkUID + } + network := &networkingv1alpha.Network{ObjectMeta: metav1.ObjectMeta{Name: internalDNSTestNetworkName, Namespace: internalDNSTestObservationNamespace, UID: networkUID}} + project := fake.NewClientBuilder().WithScheme(scheme).WithObjects(network, &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: internalDNSTestObservationNamespace, UID: "project-namespace"}}).Build() + claim := &networkingv1alpha.NetworkInterfaceClaim{ObjectMeta: metav1.ObjectMeta{Name: networkInterfaceClaimName(internalDNSTestInstanceName, defaultInterfaceName), Namespace: edgeInstance.Namespace, UID: "claim-uid"}, Spec: networkingv1alpha.NetworkInterfaceClaimSpec{Network: networkingv1alpha.LocalNetworkRef{Name: internalDNSTestNetworkName}}, Status: networkingv1alpha.NetworkInterfaceClaimStatus{NetworkInterfaceRef: &networkingv1alpha.LocalNetworkInterfaceRef{Name: iface.Name}}} + networkContext := &networkingv1alpha.NetworkContext{ObjectMeta: metav1.ObjectMeta{Name: "context", Namespace: edgeInstance.Namespace, UID: "network-context", Labels: map[string]string{networkingv1alpha.NetworkUIDLabel: "network-uid"}}, Spec: networkingv1alpha.NetworkContextSpec{Network: networkingv1alpha.LocalNetworkRef{Name: internalDNSTestNetworkName}, Location: networkingv1alpha.LocationReference{Name: "dfw"}}} + source := fake.NewClientBuilder().WithScheme(scheme).WithObjects(edgeInstance, iface, claim, networkContext, pod, node).Build() + got, err := ObserveInternalDNSInterface(context.Background(), project, internalDNSTestObservationNamespace, instance, instance.Spec.NetworkInterfaces[0], InternalDNSObservationSource{Reader: source}) + if tc.wantErr { + require.Error(t, err) + } else { + require.NoError(t, err) + } + require.Equal(t, tc.eligible, got.Eligible) + if tc.eligible { + require.Equal(t, iface.Spec.Addresses, got.Addresses) + } + }) + } +} + +func TestObserveInternalDNSInterfaceRejectsMissingSource(t *testing.T) { + _, err := ObserveInternalDNSInterface(context.Background(), nil, internalDNSTestObservationNamespace, &computev1alpha.Instance{}, computev1alpha.InstanceNetworkInterface{}, InternalDNSObservationSource{}) + require.Error(t, err) +} diff --git a/internal/controller/internaldns_publication_test.go b/internal/controller/internaldns_publication_test.go new file mode 100644 index 00000000..3384faa2 --- /dev/null +++ b/internal/controller/internaldns_publication_test.go @@ -0,0 +1,95 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package controller + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" + computev1alpha "go.datum.net/compute/api/v1alpha" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" +) + +func TestUnavailableObservationLeavesPublishedStateUntouched(t *testing.T) { + s := runtime.NewScheme() + s.AddKnownTypeWithName(dnsContributionGVK, &unstructured.Unstructured{}) + published := newDNSObject(dnsContributionGVK, internalDNSTestNamespace, "record") + published.SetUID("contribution") + published.Object[internalDNSSpecField] = map[string]any{"recordSets": []any{map[string]any{internalDNSRecordTypeField: "A", internalDNSRecordsField: []any{map[string]any{internalDNSNameField: "api", "a": map[string]any{internalDNSContentField: "10.0.0.8"}}}}}} + published.Object[internalDNSSpecField].(map[string]any)[internalDNSEligibleField] = true + published.Object[internalDNSStatusField] = map[string]any{internalDNSConditionsField: []any{map[string]any{internalDNSConditionTypeField: internalDNSConditionPublished, internalDNSStatusField: string(metav1.ConditionTrue)}}} + cl := fake.NewClientBuilder().WithScheme(s).WithObjects(published).WithStatusSubresource(published).Build() + r := &InternalDNSPublisherReconciler{} + instance := &computev1alpha.Instance{ObjectMeta: metav1.ObjectMeta{Name: "api", UID: internalDNSTestInstanceUID}} + _, err := r.reconcileAttachment(context.Background(), InternalDNSProjectAccess{Reader: cl, Writer: cl}, InternalDNSProjectIdentity{}, internalDNSTestNamespace, instance, metav1.OwnerReference{}, instanceDNSAttachment{ObservationError: context.DeadlineExceeded}, "record") + require.ErrorIs(t, err, context.DeadlineExceeded) + current := newDNSObject(dnsContributionGVK, internalDNSTestNamespace, "record") + require.NoError(t, cl.Get(context.Background(), client.ObjectKeyFromObject(published), current)) + require.Equal(t, published.Object, current.Object) +} + +func TestAllocatedInstanceDNSName(t *testing.T) { + first := &computev1alpha.Instance{ObjectMeta: metav1.ObjectMeta{Name: "Web_01", UID: "first"}} + second := first.DeepCopy() + second.UID = "second" + require.Equal(t, "web-01", allocatedInstanceDNSName(first)) + require.Equal(t, allocatedInstanceDNSName(first), allocatedInstanceDNSName(second)) + require.Equal(t, allocatedInstanceDNSName(first), allocatedInstanceDNSName(first.DeepCopy())) +} + +func TestPublisherCreatesRegistrationAndContribution(t *testing.T) { + scheme := runtime.NewScheme() + for _, gvk := range []schema.GroupVersionKind{dnsResolverContextGVK, dnsRegistrationGVK, dnsContributionGVK} { + scheme.AddKnownTypeWithName(gvk, &unstructured.Unstructured{}) + scheme.AddKnownTypeWithName(gvk.GroupVersion().WithKind(gvk.Kind+"List"), &unstructured.UnstructuredList{}) + } + resolver := resolverContextObject("context", internalDNSTestProjectUID, internalDNSTestVPCUID, true) + cl := fake.NewClientBuilder().WithScheme(scheme).WithObjects(resolver).WithStatusSubresource(newDNSObject(dnsContributionGVK, internalDNSTestNamespace, "record")).WithInterceptorFuncs(interceptor.Funcs{ + Create: func(ctx context.Context, c client.WithWatch, obj client.Object, opts ...client.CreateOption) error { + obj.SetUID(types.UID(obj.GetName() + "-uid")) + obj.SetGeneration(1) + return c.Create(ctx, obj, opts...) + }, + }).Build() + reconciler := &InternalDNSPublisherReconciler{} + instance := &computev1alpha.Instance{ObjectMeta: metav1.ObjectMeta{Name: internalDNSTestInstanceName, Namespace: internalDNSTestNamespace, UID: internalDNSTestInstanceUID}} + owner := metav1.OwnerReference{APIVersion: computev1alpha.GroupVersion.String(), Kind: internalDNSInstanceKind, Name: instance.Name, UID: instance.UID, Controller: ptrBool(true)} + access := InternalDNSProjectAccess{Reader: cl, Writer: cl} + attachment := instanceDNSAttachment{VPCUID: internalDNSTestVPCUID, Eligible: true, RecordSets: []any{map[string]any{internalDNSRecordTypeField: "A", internalDNSRecordsField: []any{map[string]any{internalDNSNameField: "", "a": map[string]any{internalDNSContentField: "10.0.0.8"}}}}}} + ready := false + for range 3 { + var err error + ready, err = reconciler.reconcileAttachment(context.Background(), access, InternalDNSProjectIdentity{ProjectUID: internalDNSTestProjectUID}, internalDNSTestNamespace, instance, owner, attachment, "record") + require.NoError(t, err) + } + require.True(t, ready) + contribution := newDNSObject(dnsContributionGVK, internalDNSTestNamespace, "record") + require.NoError(t, cl.Get(context.Background(), client.ObjectKeyFromObject(contribution), contribution)) + eligible, _, _ := unstructured.NestedBool(contribution.Object, internalDNSSpecField, internalDNSEligibleField) + require.True(t, eligible) + reason, _, _ := unstructured.NestedString(contribution.Object, internalDNSSpecField, internalDNSReasonField) + require.Equal(t, "AddressAllocatedAndProgrammed", reason) + + contribution.Object[internalDNSStatusField] = map[string]any{internalDNSConditionsField: []any{map[string]any{internalDNSConditionTypeField: internalDNSConditionPublished, internalDNSStatusField: string(metav1.ConditionTrue)}}} + require.NoError(t, cl.Status().Update(context.Background(), contribution)) + publishedStatus := contribution.Object[internalDNSStatusField] + attachment.Eligible = false + attachment.RecordSets = []any{map[string]any{internalDNSRecordTypeField: "A", internalDNSRecordsField: []any{map[string]any{internalDNSNameField: "", "a": map[string]any{internalDNSContentField: "10.0.0.9"}}}}} + _, err := reconciler.reconcileAttachment(context.Background(), access, InternalDNSProjectIdentity{ProjectUID: internalDNSTestProjectUID}, internalDNSTestNamespace, instance, owner, attachment, "record") + require.NoError(t, err) + require.NoError(t, cl.Get(context.Background(), client.ObjectKeyFromObject(contribution), contribution)) + eligible, _, _ = unstructured.NestedBool(contribution.Object, internalDNSSpecField, internalDNSEligibleField) + require.False(t, eligible) + recordSets, _, _ := unstructured.NestedSlice(contribution.Object, internalDNSSpecField, internalDNSRecordSetsField) + record := recordSets[0].(map[string]any)[internalDNSRecordsField].([]any)[0].(map[string]any) + require.Equal(t, "10.0.0.9", record["a"].(map[string]any)[internalDNSContentField]) + require.Equal(t, publishedStatus, contribution.Object[internalDNSStatusField]) +} diff --git a/internal/controller/internaldns_publisher.go b/internal/controller/internaldns_publisher.go new file mode 100644 index 00000000..f66874aa --- /dev/null +++ b/internal/controller/internaldns_publisher.go @@ -0,0 +1,578 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package controller + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "net/netip" + "reflect" + "sort" + "strings" + "time" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/validation" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/cluster" + ctrl "sigs.k8s.io/controller-runtime/pkg/reconcile" + mcbuilder "sigs.k8s.io/multicluster-runtime/pkg/builder" + mccontroller "sigs.k8s.io/multicluster-runtime/pkg/controller" + mcmanager "sigs.k8s.io/multicluster-runtime/pkg/manager" + "sigs.k8s.io/multicluster-runtime/pkg/multicluster" + mcreconcile "sigs.k8s.io/multicluster-runtime/pkg/reconcile" + + computev1alpha "go.datum.net/compute/api/v1alpha" + networkingv1alpha "go.datum.net/network-services-operator/api/v1alpha" + resourcemanagerv1alpha1 "go.miloapis.com/milo/pkg/apis/resourcemanager/v1alpha1" +) + +const ( + internalDNSEligibleField = "eligible" + internalDNSContentField = "content" + internalDNSRecordTypeField = "recordType" + internalDNSRecordsField = "records" + internalDNSInstanceKind = "Instance" + internalDNSRecordTypeAAAA = "AAAA" + internalDNSNameField = "name" + internalDNSStatusField = "status" + internalDNSReasonField = "reason" + internalDNSUIDField = "uid" + internalDNSZoneRefField = "dnsZoneRef" + internalDNSConditionAccepted = "Accepted" + internalDNSGroup = "dns.networking.miloapis.com" + internalDNSVersion = "v1alpha1" + internalDNSInstanceUID = "internal-dns.compute.datumapis.com/instance-uid" + internalDNSVPCUID = "internal-dns.compute.datumapis.com/vpc-uid" + internalDNSManagedBy = "internal-dns.compute.datumapis.com/managed-by" + internalDNSManager = "compute-instance-publisher" + internalDNSObservationInterval = 30 * time.Second + internalDNSPendingWait = 10 * time.Second + internalDNSAPITimeout = 10 * time.Second + defaultDNSConcurrency = 4 + internalDNSConditionsField = "conditions" + internalDNSObservedGenerationField = "observedGeneration" + internalDNSRegistrationRefField = "registrationRef" + internalDNSRecordTypesField = "recordTypes" + internalDNSConditionReady = "Ready" + internalDNSConditionPublished = "Published" + internalDNSConditionTypeField = "type" + internalDNSSpecField = "spec" + internalDNSRecordSetsField = "recordSets" +) + +var ( + dnsResolverContextGVK = schema.GroupVersionKind{Group: internalDNSGroup, Version: internalDNSVersion, Kind: "DNSResolverContext"} + dnsRegistrationGVK = schema.GroupVersionKind{Group: internalDNSGroup, Version: internalDNSVersion, Kind: "DNSRegistration"} + dnsContributionGVK = schema.GroupVersionKind{Group: internalDNSGroup, Version: internalDNSVersion, Kind: "DNSRecordContribution"} +) + +// InternalDNSProjectIdentity identifies the project that owns DNS resources. +type InternalDNSProjectIdentity struct { + ProjectName string + ProjectUID types.UID +} + +// InternalDNSProjectAccess supplies the owning project API reader and writer. +type InternalDNSProjectAccess struct { + Reader client.Reader + Writer client.Client +} + +type InternalDNSProjectAccessFunc func( + ctx context.Context, + projectID string, + providerCluster cluster.Cluster, +) (InternalDNSProjectAccess, error) + +type internalDNSClusterGetter interface { + GetCluster(ctx context.Context, clusterName multicluster.ClusterName) (cluster.Cluster, error) +} + +// InternalDNSPublisherReconciler publishes the private addresses of an Instance +// under one stable identity name per attached VPC. DNS failures only requeue +// this controller; they never alter Compute readiness. +type InternalDNSPublisherReconciler struct { + mgr internalDNSClusterGetter + ProjectIDForInstance func(context.Context, multicluster.ClusterName, *computev1alpha.Instance) (string, error) + ProjectNamespaceForInstance func(context.Context, multicluster.ClusterName, *computev1alpha.Instance) (string, error) + ProjectAccess InternalDNSProjectAccessFunc + ResolveProjectIdentity func(context.Context, string) (InternalDNSProjectIdentity, error) + ProjectIdentities map[string]InternalDNSProjectIdentity + MaxConcurrentReconciles int + Now func() time.Time + ObservationSources map[string]InternalDNSObservationSource + ObserveInterface InternalDNSObserveFunc +} + +func (r *InternalDNSPublisherReconciler) Reconcile(ctx context.Context, req mcreconcile.Request) (ctrl.Result, error) { + ctx, cancel := context.WithTimeout(ctx, internalDNSAPITimeout) + defer cancel() + + providerCluster, err := r.mgr.GetCluster(ctx, req.ClusterName) + if err != nil { + return ctrl.Result{}, err + } + + var instance computev1alpha.Instance + if err := providerCluster.GetAPIReader().Get(ctx, req.NamespacedName, &instance); err != nil { + return ctrl.Result{}, client.IgnoreNotFound(err) + } + + projectID, err := r.projectID(ctx, req.ClusterName, &instance) + if err != nil { + return ctrl.Result{}, fmt.Errorf("resolve DNS project identity: %w", err) + } + projectNamespace, err := r.projectNamespace(ctx, req.ClusterName, &instance) + if err != nil { + return ctrl.Result{}, fmt.Errorf("resolve DNS project namespace: %w", err) + } + identity, err := r.projectIdentity(ctx, projectID) + if err != nil { + return ctrl.Result{}, err + } + access, err := r.projectAccess(ctx, projectID, providerCluster) + if err != nil { + return ctrl.Result{}, err + } + // DNS resources live in the project API and are owned by the project-side + // Instance lifetime, even when this observation came from an edge copy with + // a different Kubernetes UID. This makes normal deletion and replacement + // cleanup independent of receiving a final reconcile event. + var projectInstance computev1alpha.Instance + if err := access.Reader.Get(ctx, client.ObjectKey{Namespace: projectNamespace, Name: instance.Name}, &projectInstance); err != nil { + return ctrl.Result{}, fmt.Errorf("resolve project Instance owner for DNS publication: %w", err) + } + if projectInstance.UID == "" { + return ctrl.Result{}, fmt.Errorf("project Instance %s/%s has no API-assigned UID", projectNamespace, instance.Name) + } + if projectInstance.UID != instance.UID { + return ctrl.Result{}, fmt.Errorf("refusing DNS publication from Instance copy UID %q for current project Instance UID %q", instance.UID, projectInstance.UID) + } + owner := metav1.OwnerReference{ + APIVersion: computev1alpha.GroupVersion.String(), + Kind: internalDNSInstanceKind, + Name: projectInstance.Name, + UID: projectInstance.UID, + Controller: ptrBool(true), + } + + if !instance.DeletionTimestamp.IsZero() { + return ctrl.Result{}, r.deletePublishedResources(ctx, access, projectNamespace, projectInstance.UID, nil) + } + + attachments, err := instanceDNSAttachments(ctx, access.Reader, projectNamespace, &projectInstance, r.observeInterface) + if err != nil { + return ctrl.Result{}, err + } + desired := make(map[string]struct{}, len(attachments)) + publicationStatus := make([]computev1alpha.InstanceDNSStatus, 0, len(attachments)) + var publicationErr error + pending := false + for _, attachment := range attachments { + resourceName := internalDNSResourceName(projectInstance.UID, attachment.VPCUID) + desired[resourceName] = struct{}{} + ready, err := r.reconcileAttachment(ctx, access, identity, projectNamespace, &projectInstance, owner, attachment, resourceName) + publication, statusErr := r.publicationStatus(ctx, access.Reader, identity, projectNamespace, &projectInstance, attachment, resourceName, ready, err) + publicationStatus = append(publicationStatus, publication) + if statusErr != nil { + publicationErr = errors.Join(publicationErr, statusErr) + } + if err != nil { + publicationErr = errors.Join(publicationErr, err) + } + pending = pending || !ready + } + if err := r.reportPublication(ctx, access.Writer, &projectInstance, publicationStatus); err != nil { + return ctrl.Result{}, err + } + if err := r.deletePublishedResources(ctx, access, projectNamespace, projectInstance.UID, desired); err != nil { + return ctrl.Result{}, err + } + if pending { + return ctrl.Result{RequeueAfter: internalDNSPendingWait}, publicationErr + } + return ctrl.Result{RequeueAfter: internalDNSObservationInterval}, publicationErr +} + +type instanceDNSAttachment struct { + VPCUID types.UID + Network networkingv1alpha.NetworkRef + RecordSets []any + Eligible bool + ObservationError error +} + +func instanceDNSAttachments(ctx context.Context, reader client.Reader, projectNamespace string, instance *computev1alpha.Instance, observe InternalDNSObserveFunc) ([]instanceDNSAttachment, error) { + byVPC := map[types.UID]*instanceDNSAttachment{} + for _, nic := range instance.Spec.NetworkInterfaces { + namespace := nic.Network.Namespace + if namespace == "" { + namespace = projectNamespace + } + var network networkingv1alpha.Network + if err := reader.Get(ctx, client.ObjectKey{Namespace: namespace, Name: nic.Network.Name}, &network); err != nil { + return nil, fmt.Errorf("resolve VPC %s/%s for instance DNS: %w", namespace, nic.Network.Name, err) + } + if network.UID == "" { + return nil, fmt.Errorf("network %s/%s has no API-assigned UID", namespace, nic.Network.Name) + } + attachment := byVPC[network.UID] + if attachment == nil { + attachment = &instanceDNSAttachment{VPCUID: network.UID, Network: networkingv1alpha.NetworkRef{Name: network.Name, Namespace: namespace}, RecordSets: []any{}} + byVPC[network.UID] = attachment + } + + if observe == nil { + attachment.ObservationError = fmt.Errorf("no current interface observation is available") + continue + } + observation, err := observe(ctx, reader, projectNamespace, instance, nic) + if err != nil { + attachment.ObservationError = errors.Join(attachment.ObservationError, err) + continue + } + if !observation.Eligible { + continue + } + attachment.Eligible = true + for _, address := range observation.Addresses { + addr, ok := hostAddress(address.Address) + if !ok { + continue + } + rrtype, field := "A", "a" + if addr.Is6() { + rrtype, field = internalDNSRecordTypeAAAA, "aaaa" + } + attachment.RecordSets = append(attachment.RecordSets, map[string]any{ + internalDNSRecordTypeField: rrtype, + internalDNSRecordsField: []any{map[string]any{internalDNSNameField: "", "ttl": int64(30), field: map[string]any{internalDNSContentField: addr.String()}}}, + }) + } + } + + attachments := make([]instanceDNSAttachment, 0, len(byVPC)) + for _, attachment := range byVPC { + attachment.Eligible = attachment.Eligible && len(attachment.RecordSets) > 0 + sort.Slice(attachment.RecordSets, func(i, j int) bool { + return fmt.Sprint(attachment.RecordSets[i]) < fmt.Sprint(attachment.RecordSets[j]) + }) + attachments = append(attachments, *attachment) + } + sort.Slice(attachments, func(i, j int) bool { return attachments[i].VPCUID < attachments[j].VPCUID }) + return attachments, nil +} + +func internalDNSInterfaceName(nic computev1alpha.InstanceNetworkInterface) string { + if nic.Name != "" { + return nic.Name + } + return defaultInterfaceName +} + +func hostAddress(raw string) (netip.Addr, bool) { + if addr, err := netip.ParseAddr(raw); err == nil { + return addr.Unmap(), true + } + prefix, err := netip.ParsePrefix(raw) + if err != nil || prefix.Bits() != prefix.Addr().BitLen() { + return netip.Addr{}, false + } + return prefix.Addr().Unmap(), true +} + +func (r *InternalDNSPublisherReconciler) reconcileAttachment(ctx context.Context, access InternalDNSProjectAccess, identity InternalDNSProjectIdentity, namespace string, instance *computev1alpha.Instance, owner metav1.OwnerReference, attachment instanceDNSAttachment, resourceName string) (bool, error) { + if attachment.ObservationError != nil { + return false, attachment.ObservationError + } + managed, err := findResolverContext(ctx, access.Reader, namespace, identity.ProjectUID, attachment.VPCUID) + if err != nil { + return false, err + } + if managed == nil { + return false, nil + } + _, zoneRef, err := allocatedDNSNamespace(managed) + if err != nil { + return false, err + } + + labels := map[string]string{ + internalDNSInstanceUID: string(instance.UID), + internalDNSVPCUID: string(attachment.VPCUID), + internalDNSManagedBy: internalDNSManager, + } + ownerName := allocatedInstanceDNSName(instance) + for i := range attachment.RecordSets { + rrset := attachment.RecordSets[i].(map[string]any) + records := rrset[internalDNSRecordsField].([]any) + for _, record := range records { + record.(map[string]any)[internalDNSNameField] = ownerName + } + } + + registrationSpec := map[string]any{ + internalDNSZoneRefField: zoneRef, + internalDNSNameField: ownerName, + internalDNSRecordTypesField: []any{"A", internalDNSRecordTypeAAAA}, + "ttlSeconds": int64(30), + } + registration, changed, err := ensureDNSObject(ctx, access, dnsRegistrationGVK, namespace, resourceName, labels, owner, registrationSpec) + if err != nil || changed { + return false, err + } + registrationRef := objectReference(registration) + + contributionSpec := map[string]any{ + internalDNSRegistrationRefField: registrationRef, + internalDNSRecordSetsField: attachment.RecordSets, + internalDNSEligibleField: attachment.Eligible, + internalDNSReasonField: map[bool]string{true: "AddressAllocatedAndProgrammed", false: "AddressUnavailable"}[attachment.Eligible], + } + _, changed, err = ensureDNSObject(ctx, access, dnsContributionGVK, namespace, resourceName, labels, owner, contributionSpec) + return err == nil && !changed, err +} + +func findResolverContext(ctx context.Context, reader client.Reader, namespace string, projectUID, vpcUID types.UID) (*unstructured.Unstructured, error) { + list := &unstructured.UnstructuredList{} + list.SetGroupVersionKind(dnsResolverContextGVK.GroupVersion().WithKind(dnsResolverContextGVK.Kind + "List")) + if err := reader.List(ctx, list, client.InNamespace(namespace)); err != nil { + return nil, fmt.Errorf("list DNS resolver contexts: %w", err) + } + var match *unstructured.Unstructured + for i := range list.Items { + item := &list.Items[i] + if !item.GetDeletionTimestamp().IsZero() || item.GetUID() == "" || item.GetGeneration() < 1 { + continue + } + consumerID, _, _ := unstructured.NestedString(item.Object, internalDNSSpecField, "consumerID") + if consumerID != string(projectUID)+"/"+string(vpcUID) || !unstructuredConditionCurrent(item, internalDNSConditionReady) { + continue + } + if match != nil { + return nil, fmt.Errorf("multiple ready DNS resolver contexts match project UID %q and VPC UID %q", projectUID, vpcUID) + } + match = item.DeepCopy() + } + return match, nil +} + +func allocatedDNSNamespace(resolver *unstructured.Unstructured) (string, map[string]any, error) { + suffix, _, _ := unstructured.NestedString(resolver.Object, internalDNSStatusField, "managedNamespace", "suffix") + suffix = strings.TrimSuffix(strings.ToLower(suffix), ".") + zoneRef, _, _ := unstructured.NestedMap(resolver.Object, internalDNSStatusField, "managedNamespace", internalDNSZoneRefField) + if len(validation.IsDNS1123Subdomain(suffix)) > 0 || referenceName(zoneRef) == "" { + return "", nil, fmt.Errorf("DNS context %q has no valid allocated namespace", resolver.GetName()) + } + return suffix, map[string]any{internalDNSNameField: referenceName(zoneRef)}, nil +} + +func unstructuredConditionCurrent(obj *unstructured.Unstructured, conditionType string) bool { + conditions, _, _ := unstructured.NestedSlice(obj.Object, internalDNSStatusField, internalDNSConditionsField) + for _, raw := range conditions { + condition, ok := raw.(map[string]any) + if ok && condition[internalDNSConditionTypeField] == conditionType && condition[internalDNSStatusField] == string(metav1.ConditionTrue) && condition[internalDNSObservedGenerationField] == obj.GetGeneration() { + return true + } + } + return false +} + +func ensureDNSObject(ctx context.Context, access InternalDNSProjectAccess, gvk schema.GroupVersionKind, namespace, name string, labels map[string]string, owner metav1.OwnerReference, spec map[string]any) (*unstructured.Unstructured, bool, error) { + desired := newDNSObject(gvk, namespace, name) + desired.SetLabels(labels) + desired.SetOwnerReferences([]metav1.OwnerReference{owner}) + desired.Object[internalDNSSpecField] = spec + current := newDNSObject(gvk, namespace, name) + err := access.Reader.Get(ctx, client.ObjectKeyFromObject(desired), current) + if apierrors.IsNotFound(err) { + if err := access.Writer.Create(ctx, desired); err != nil { + return nil, false, fmt.Errorf("create %s %s/%s: %w", gvk.Kind, namespace, name, err) + } + return desired, true, nil + } + if err != nil { + return nil, false, err + } + if !current.GetDeletionTimestamp().IsZero() { + return nil, false, fmt.Errorf("%s %s/%s is terminating", gvk.Kind, namespace, name) + } + if current.GetLabels()[internalDNSManagedBy] != internalDNSManager || current.GetLabels()[internalDNSInstanceUID] != labels[internalDNSInstanceUID] || !ownedBy(current, owner) { + return nil, false, fmt.Errorf("refusing to adopt foreign %s %s/%s", gvk.Kind, namespace, name) + } + currentSpec, _, _ := unstructured.NestedMap(current.Object, internalDNSSpecField) + if authorityReferencesChanged(gvk, currentSpec, spec) { + if err := access.Writer.Delete(ctx, current); client.IgnoreNotFound(err) != nil { + return nil, false, err + } + return current, true, nil + } + if !reflect.DeepEqual(currentSpec, spec) || !reflect.DeepEqual(current.GetLabels(), labels) || !reflect.DeepEqual(current.GetOwnerReferences(), []metav1.OwnerReference{owner}) { + current.Object[internalDNSSpecField] = spec + current.SetLabels(labels) + current.SetOwnerReferences([]metav1.OwnerReference{owner}) + if err := access.Writer.Update(ctx, current); err != nil { + return nil, false, fmt.Errorf("update %s %s/%s: %w", gvk.Kind, namespace, name, err) + } + return current, true, nil + } + return current, false, nil +} + +func ownedBy(obj metav1.Object, owner metav1.OwnerReference) bool { + for _, ref := range obj.GetOwnerReferences() { + if ref.UID == owner.UID && ref.APIVersion == owner.APIVersion && ref.Kind == owner.Kind && ref.Name == owner.Name { + return true + } + } + return false +} + +func ptrBool(value bool) *bool { return &value } + +func authorityReferencesChanged(gvk schema.GroupVersionKind, current, desired map[string]any) bool { + if gvk != dnsContributionGVK { + return false + } + return !reflect.DeepEqual(current[internalDNSRegistrationRefField], desired[internalDNSRegistrationRefField]) +} + +func (r *InternalDNSPublisherReconciler) deletePublishedResources(ctx context.Context, access InternalDNSProjectAccess, namespace string, instanceUID types.UID, keep map[string]struct{}) error { + selector := client.MatchingLabels{internalDNSInstanceUID: string(instanceUID), internalDNSManagedBy: internalDNSManager} + for _, gvk := range []schema.GroupVersionKind{dnsContributionGVK, dnsRegistrationGVK} { + list := &unstructured.UnstructuredList{} + list.SetGroupVersionKind(gvk.GroupVersion().WithKind(gvk.Kind + "List")) + if err := access.Reader.List(ctx, list, client.InNamespace(namespace), selector); err != nil { + return fmt.Errorf("list stale %s resources: %w", gvk.Kind, err) + } + for i := range list.Items { + item := &list.Items[i] + if _, retained := keep[item.GetName()]; retained { + continue + } + if err := access.Writer.Delete(ctx, item); client.IgnoreNotFound(err) != nil { + return fmt.Errorf("delete stale %s %s/%s: %w", gvk.Kind, namespace, item.GetName(), err) + } + } + } + return nil +} + +func newDNSObject(gvk schema.GroupVersionKind, namespace, name string) *unstructured.Unstructured { + obj := &unstructured.Unstructured{} + obj.SetGroupVersionKind(gvk) + obj.SetNamespace(namespace) + obj.SetName(name) + return obj +} + +func objectReference(obj *unstructured.Unstructured) map[string]any { + ref := map[string]any{internalDNSNameField: obj.GetName(), internalDNSUIDField: string(obj.GetUID())} + ref["generation"] = obj.GetGeneration() + return ref +} + +func referenceName(values map[string]any) string { + value, _ := values[internalDNSNameField].(string) + return value +} + +func allocatedInstanceDNSName(instance *computev1alpha.Instance) string { + base := strings.ToLower(instance.Name) + var b strings.Builder + for _, r := range base { + if (r >= 'a' && r <= 'z') || (r >= '0' && r <= '9') || r == '-' { + b.WriteRune(r) + } else { + b.WriteByte('-') + } + } + base = strings.Trim(b.String(), "-") + if base == "" { + base = "instance" + } + if len(base) > 63 { + return strings.TrimRight(base[:46], "-") + "-" + shortHash(instance.Name) + } + return base +} + +func internalDNSResourceName(instanceUID, vpcUID types.UID) string { + return "instance-" + shortHash(string(instanceUID)+"/"+string(vpcUID)) +} + +func shortHash(value string) string { + sum := sha256.Sum256([]byte(value)) + return hex.EncodeToString(sum[:])[:16] +} + +func (r *InternalDNSPublisherReconciler) projectID(ctx context.Context, clusterName multicluster.ClusterName, instance *computev1alpha.Instance) (string, error) { + if r.ProjectIDForInstance != nil { + return r.ProjectIDForInstance(ctx, clusterName, instance) + } + return string(clusterName), nil +} + +func (r *InternalDNSPublisherReconciler) projectNamespace(ctx context.Context, clusterName multicluster.ClusterName, instance *computev1alpha.Instance) (string, error) { + if r.ProjectNamespaceForInstance != nil { + return r.ProjectNamespaceForInstance(ctx, clusterName, instance) + } + return instance.Namespace, nil +} + +func (r *InternalDNSPublisherReconciler) projectAccess(ctx context.Context, projectID string, providerCluster cluster.Cluster) (InternalDNSProjectAccess, error) { + if r.ProjectAccess != nil { + return r.ProjectAccess(ctx, projectID, providerCluster) + } + return InternalDNSProjectAccess{Reader: providerCluster.GetAPIReader(), Writer: providerCluster.GetClient()}, nil +} + +// InternalDNSProjectIdentityResolver reads the current project identity from Milo. +// Unavailable or terminating projects stop publication without withdrawing records. +func InternalDNSProjectIdentityResolver(reader client.Reader) func(context.Context, string) (InternalDNSProjectIdentity, error) { + return func(ctx context.Context, projectName string) (InternalDNSProjectIdentity, error) { + var project resourcemanagerv1alpha1.Project + if err := reader.Get(ctx, client.ObjectKey{Name: projectName}, &project); err != nil { + return InternalDNSProjectIdentity{}, fmt.Errorf("read DNS project %q: %w", projectName, err) + } + if project.UID == "" || !project.DeletionTimestamp.IsZero() { + return InternalDNSProjectIdentity{}, fmt.Errorf("DNS project %q has no active identity", projectName) + } + return InternalDNSProjectIdentity{ProjectName: project.Name, ProjectUID: project.UID}, nil + } +} + +func (r *InternalDNSPublisherReconciler) projectIdentity(ctx context.Context, projectName string) (InternalDNSProjectIdentity, error) { + if r.ResolveProjectIdentity != nil { + return r.ResolveProjectIdentity(ctx, projectName) + } + identity, ok := r.ProjectIdentities[projectName] + if !ok || identity.ProjectUID == "" { + return InternalDNSProjectIdentity{}, fmt.Errorf("internal DNS identity is not configured for project %q", projectName) + } + return identity, nil +} + +func (r *InternalDNSPublisherReconciler) SetupWithManager(mgr mcmanager.Manager) error { + if r.ResolveProjectIdentity == nil && len(r.ProjectIdentities) == 0 { + return errors.New("an internal DNS project identity resolver is required") + } + r.mgr = mgr + concurrency := r.MaxConcurrentReconciles + if concurrency <= 0 { + concurrency = defaultDNSConcurrency + } + return mcbuilder.ControllerManagedBy(mgr). + Named("internal-dns-instance-publisher"). + For(&computev1alpha.Instance{}, mcbuilder.WithEngageWithLocalCluster(false)). + WithOptions(mccontroller.Options{MaxConcurrentReconciles: concurrency}). + Complete(r) +} diff --git a/internal/controller/internaldns_publisher_test.go b/internal/controller/internaldns_publisher_test.go new file mode 100644 index 00000000..cc3d32a8 --- /dev/null +++ b/internal/controller/internaldns_publisher_test.go @@ -0,0 +1,228 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package controller + +import ( + "context" + "errors" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + computev1alpha "go.datum.net/compute/api/v1alpha" + networkingv1alpha "go.datum.net/network-services-operator/api/v1alpha" + resourcemanagerv1alpha1 "go.miloapis.com/milo/pkg/apis/resourcemanager/v1alpha1" +) + +const ( + internalDNSTestDiscoveredProjectName = "project-b" + internalDNSTestProjectUID = "project-a" + internalDNSTestNamespace = "dns-test" + internalDNSTestDefaultDomain = "datum.internal" + internalDNSTestInstanceName = "api" + internalDNSTestInstanceUID = "dns-instance-uid" + internalDNSTestEdgeUID = "dns-edge-instance" + internalDNSTestForeignValue = "dns-unauthorized" + internalDNSTestSubject = "system:serviceaccount:dns-test:publisher" + internalDNSTestObservationNamespace = "dns-observation-project" + internalDNSTestNetworkName = "dns-application" + internalDNSTestVPCUID = "dns-test-vpc" + internalDNSTestSourceAPIUID = "dns-project-api" +) + +func TestInstanceDNSAttachmentsMapsTenancyAndAddressFamilies(t *testing.T) { + scheme := runtime.NewScheme() + require.NoError(t, networkingv1alpha.AddToScheme(scheme)) + network := &networkingv1alpha.Network{ObjectMeta: metav1.ObjectMeta{Name: "prod", Namespace: internalDNSTestNamespace, UID: "vpc-uid"}} + cl := fake.NewClientBuilder().WithScheme(scheme).WithObjects(network).Build() + instance := &computev1alpha.Instance{ + ObjectMeta: metav1.ObjectMeta{Name: internalDNSTestInstanceName, Namespace: internalDNSTestNamespace, UID: internalDNSTestInstanceUID}, + Spec: computev1alpha.InstanceSpec{NetworkInterfaces: []computev1alpha.InstanceNetworkInterface{{Name: defaultInterfaceName, Network: networkingv1alpha.NetworkRef{Name: "prod"}}}}, + Status: computev1alpha.InstanceStatus{ + // Application readiness is deliberately false. Instance identity is + // governed by the interface allocation/programming lifecycle. + Conditions: []metav1.Condition{{Type: computev1alpha.InstanceReady, Status: metav1.ConditionFalse}}, + NetworkInterfaces: []computev1alpha.InstanceNetworkInterfaceStatus{{ + Name: defaultInterfaceName, + Addresses: []computev1alpha.InstanceNetworkInterfaceAddress{ + {Address: "10.0.0.8/32"}, + {Address: "2001:db8::8/128"}, + {Address: "2001:db8:1::/96"}, // delegated block is not one host identity + }, + Conditions: []metav1.Condition{ + {Type: computev1alpha.InstanceNetworkInterfaceAllocated, Status: metav1.ConditionTrue}, + {Type: computev1alpha.InstanceNetworkInterfaceProgrammed, Status: metav1.ConditionTrue}, + }, + }}, + }, + } + + eligible := true + observation := func(context.Context, client.Reader, string, *computev1alpha.Instance, computev1alpha.InstanceNetworkInterface) (InternalDNSObservedInterface, error) { + return InternalDNSObservedInterface{Addresses: []networkingv1alpha.NetworkInterfaceAddress{{Address: "10.0.0.8/32"}, {Address: "2001:db8::8/128"}, {Address: "2001:db8:1::/96"}}, Eligible: eligible}, nil + } + attachments, err := instanceDNSAttachments(context.Background(), cl, internalDNSTestNamespace, instance, observation) + require.NoError(t, err) + require.Len(t, attachments, 1) + assert.Equal(t, types.UID("vpc-uid"), attachments[0].VPCUID) + assert.True(t, attachments[0].Eligible) + assert.Len(t, attachments[0].RecordSets, 2, "A and AAAA host addresses should be retained") + + eligible = false + attachments, err = instanceDNSAttachments(context.Background(), cl, internalDNSTestNamespace, instance, observation) + require.NoError(t, err) + require.Len(t, attachments, 1) + assert.False(t, attachments[0].Eligible) + assert.NotNil(t, attachments[0].RecordSets) + assert.Empty(t, attachments[0].RecordSets) +} + +func TestFindResolverContextRequiresProjectAndVPCUID(t *testing.T) { + s := runtime.NewScheme() + s.AddKnownTypeWithName(dnsResolverContextGVK, &unstructured.Unstructured{}) + s.AddKnownTypeWithName(dnsResolverContextGVK.GroupVersion().WithKind("DNSResolverContextList"), &unstructured.UnstructuredList{}) + wrong := resolverContextObject("wrong", "other-project", internalDNSTestVPCUID, true) + right := resolverContextObject("right", "project-a", internalDNSTestVPCUID, true) + wrongVPC := resolverContextObject("wrong-vpc", "project-a", "vpc-b", true) + pending := resolverContextObject("pending", "project-a", internalDNSTestVPCUID, false) + cl := fake.NewClientBuilder().WithScheme(s).WithObjects(wrong, wrongVPC, pending, right).Build() + + got, err := findResolverContext(context.Background(), cl, internalDNSTestNamespace, "project-a", internalDNSTestVPCUID) + require.NoError(t, err) + require.NotNil(t, got) + assert.Equal(t, "right", got.GetName()) + + got, err = findResolverContext(context.Background(), cl, internalDNSTestNamespace, internalDNSTestDiscoveredProjectName, internalDNSTestVPCUID) + require.NoError(t, err) + assert.Nil(t, got) +} + +func TestFindResolverContextRejectsAmbiguityAndIgnoresTerminatingBinding(t *testing.T) { + s := runtime.NewScheme() + s.AddKnownTypeWithName(dnsResolverContextGVK, &unstructured.Unstructured{}) + s.AddKnownTypeWithName(dnsResolverContextGVK.GroupVersion().WithKind("DNSResolverContextList"), &unstructured.UnstructuredList{}) + first := resolverContextObject("first", "project-a", internalDNSTestVPCUID, true) + second := resolverContextObject("second", "project-a", internalDNSTestVPCUID, true) + cl := fake.NewClientBuilder().WithScheme(s).WithObjects(first, second).Build() + + _, err := findResolverContext(context.Background(), cl, internalDNSTestNamespace, "project-a", internalDNSTestVPCUID) + require.Error(t, err) + assert.Contains(t, err.Error(), "multiple ready") + + now := metav1.Now() + second.SetDeletionTimestamp(&now) + second.SetFinalizers([]string{"test.example/finalizer"}) + cl = fake.NewClientBuilder().WithScheme(s).WithObjects(first, second).Build() + got, err := findResolverContext(context.Background(), cl, internalDNSTestNamespace, "project-a", internalDNSTestVPCUID) + require.NoError(t, err) + require.NotNil(t, got) + assert.Equal(t, "first", got.GetName()) +} + +func resolverContextObject(name, projectUID, vpcUID string, accepted bool) *unstructured.Unstructured { + status := string(metav1.ConditionFalse) + if accepted { + status = string(metav1.ConditionTrue) + } + obj := newDNSObject(dnsResolverContextGVK, internalDNSTestNamespace, name) + obj.SetUID(types.UID(name + "-uid")) + obj.SetGeneration(1) + obj.Object[internalDNSSpecField] = map[string]any{"consumerID": projectUID + "/" + vpcUID} + obj.Object[internalDNSStatusField] = map[string]any{ + internalDNSConditionsField: []any{map[string]any{internalDNSConditionTypeField: internalDNSConditionReady, internalDNSStatusField: status, internalDNSObservedGenerationField: int64(1)}}, + "managedNamespace": map[string]any{"suffix": internalDNSTestDefaultDomain, internalDNSZoneRefField: map[string]any{internalDNSNameField: "zone"}}, + } + return obj +} + +func TestEnsureDNSObjectRefusesForeignCollision(t *testing.T) { + s := runtime.NewScheme() + s.AddKnownTypeWithName(dnsRegistrationGVK, &unstructured.Unstructured{}) + s.AddKnownTypeWithName(dnsRegistrationGVK.GroupVersion().WithKind("DNSRegistrationList"), &unstructured.UnstructuredList{}) + foreign := newDNSObject(dnsRegistrationGVK, internalDNSTestNamespace, "instance-collision") + foreign.SetUID("foreign") + foreign.Object[internalDNSSpecField] = map[string]any{internalDNSNameField: "someone-else"} + cl := fake.NewClientBuilder().WithScheme(s).WithObjects(foreign).Build() + owner := metav1.OwnerReference{APIVersion: computev1alpha.GroupVersion.String(), Kind: internalDNSInstanceKind, Name: internalDNSTestInstanceName, UID: internalDNSTestInstanceUID} + + _, _, err := ensureDNSObject(context.Background(), InternalDNSProjectAccess{Reader: cl, Writer: cl}, dnsRegistrationGVK, internalDNSTestNamespace, foreign.GetName(), map[string]string{ + internalDNSManagedBy: internalDNSManager, internalDNSInstanceUID: internalDNSTestInstanceUID, + }, owner, map[string]any{internalDNSNameField: internalDNSTestInstanceName}) + require.Error(t, err) + assert.Contains(t, err.Error(), "refusing to adopt foreign") +} + +func TestEnsureDNSObjectCreatesProjectInstanceOwnershipWithoutDeleteBlock(t *testing.T) { + s := runtime.NewScheme() + s.AddKnownTypeWithName(dnsRegistrationGVK, &unstructured.Unstructured{}) + s.AddKnownTypeWithName(dnsRegistrationGVK.GroupVersion().WithKind("DNSRegistrationList"), &unstructured.UnstructuredList{}) + cl := fake.NewClientBuilder().WithScheme(s).Build() + owner := metav1.OwnerReference{APIVersion: computev1alpha.GroupVersion.String(), Kind: internalDNSInstanceKind, Name: internalDNSTestInstanceName, UID: "project-instance-uid", Controller: ptrBool(true)} + + created, changed, err := ensureDNSObject(context.Background(), InternalDNSProjectAccess{Reader: cl, Writer: cl}, dnsRegistrationGVK, internalDNSTestNamespace, "instance-owned", map[string]string{ + internalDNSManagedBy: internalDNSManager, internalDNSInstanceUID: "project-instance-uid", + }, owner, map[string]any{internalDNSNameField: internalDNSTestInstanceName}) + require.NoError(t, err) + assert.True(t, changed) + require.Len(t, created.GetOwnerReferences(), 1) + assert.Equal(t, types.UID("project-instance-uid"), created.GetOwnerReferences()[0].UID) + assert.Nil(t, created.GetOwnerReferences()[0].BlockOwnerDeletion, + "publisher must not require delete permission on the owning Instance") +} + +func TestDNSProjectIdentityDiscoversNewProjectsAndReplacementIdentities(t *testing.T) { + scheme := runtime.NewScheme() + require.NoError(t, resourcemanagerv1alpha1.AddToScheme(scheme)) + cl := fake.NewClientBuilder().WithScheme(scheme).Build() + resolve := InternalDNSProjectIdentityResolver(cl) + _, err := resolve(context.Background(), internalDNSTestDiscoveredProjectName) + require.Error(t, err) + + project := &resourcemanagerv1alpha1.Project{ObjectMeta: metav1.ObjectMeta{Name: internalDNSTestDiscoveredProjectName, UID: "project-b-first"}} + require.NoError(t, cl.Create(context.Background(), project)) + identity, err := resolve(context.Background(), project.Name) + require.NoError(t, err) + assert.Equal(t, InternalDNSProjectIdentity{ProjectName: project.Name, ProjectUID: project.UID}, identity) + + require.NoError(t, cl.Delete(context.Background(), project)) + project.ResourceVersion = "" + project.UID = "project-b-second" + require.NoError(t, cl.Create(context.Background(), project)) + identity, err = resolve(context.Background(), project.Name) + require.NoError(t, err) + assert.Equal(t, project.UID, identity.ProjectUID) +} + +func TestDNSProjectIdentityRejectsInactiveProjectsAndReadFailures(t *testing.T) { + scheme := runtime.NewScheme() + require.NoError(t, resourcemanagerv1alpha1.AddToScheme(scheme)) + now := metav1.Now() + for _, project := range []*resourcemanagerv1alpha1.Project{ + {ObjectMeta: metav1.ObjectMeta{Name: internalDNSTestDiscoveredProjectName}}, + {ObjectMeta: metav1.ObjectMeta{Name: internalDNSTestDiscoveredProjectName, UID: internalDNSTestDiscoveredProjectName, DeletionTimestamp: &now, Finalizers: []string{"test.datum.net/cleanup"}}}, + } { + cl := fake.NewClientBuilder().WithScheme(scheme).WithObjects(project).Build() + _, err := InternalDNSProjectIdentityResolver(cl)(context.Background(), project.Name) + require.Error(t, err) + } + unavailable := errors.New("project API unavailable") + reader := &unavailableDNSProjectReader{err: unavailable} + _, err := InternalDNSProjectIdentityResolver(reader)(context.Background(), internalDNSTestDiscoveredProjectName) + require.ErrorIs(t, err, unavailable) +} + +type unavailableDNSProjectReader struct { + client.Reader + err error +} + +func (r *unavailableDNSProjectReader) Get(context.Context, client.ObjectKey, client.Object, ...client.GetOption) error { + return r.err +} diff --git a/internal/controller/internaldns_status.go b/internal/controller/internaldns_status.go new file mode 100644 index 00000000..c44e3eae --- /dev/null +++ b/internal/controller/internaldns_status.go @@ -0,0 +1,93 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package controller + +import ( + "context" + "fmt" + "reflect" + "strings" + + computev1alpha "go.datum.net/compute/api/v1alpha" + apierrors "k8s.io/apimachinery/pkg/api/errors" + apimeta "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +func (r *InternalDNSPublisherReconciler) observeInterface(ctx context.Context, reader client.Reader, namespace string, instance *computev1alpha.Instance, nic computev1alpha.InstanceNetworkInterface) (InternalDNSObservedInterface, error) { + if r.ObserveInterface != nil { + return r.ObserveInterface(ctx, reader, namespace, instance, nic) + } + return ObserveInternalDNSInterface(ctx, reader, namespace, instance, nic, r.ObservationSources[instance.Labels[computev1alpha.LocationLabel]]) +} + +func (r *InternalDNSPublisherReconciler) publicationStatus(ctx context.Context, reader client.Reader, identity InternalDNSProjectIdentity, namespace string, instance *computev1alpha.Instance, attachment instanceDNSAttachment, resourceName string, observed bool, reconcileErr error) (computev1alpha.InstanceDNSStatus, error) { + result := computev1alpha.InstanceDNSStatus{Network: attachment.Network} + for _, previous := range instance.Status.DNS { + if previous.Network.Name == result.Network.Name && previous.Network.Namespace == result.Network.Namespace { + result.Conditions = append([]metav1.Condition(nil), previous.Conditions...) + } + } + condition := metav1.Condition{Type: internalDNSConditionReady, Status: metav1.ConditionFalse, Reason: "PendingPublication", Message: "Waiting for DNS publication", ObservedGeneration: instance.Generation} + registration := newDNSObject(dnsRegistrationGVK, namespace, resourceName) + err := reader.Get(ctx, client.ObjectKeyFromObject(registration), registration) + if err != nil && !apierrors.IsNotFound(err) { + return result, err + } + if err == nil { + canonical, _, _ := unstructured.NestedString(registration.Object, internalDNSStatusField, "canonicalFQDN") + canonical = strings.TrimSuffix(strings.ToLower(canonical), ".") + matches, err := publicationMatchesContext(ctx, reader, identity, namespace, instance, attachment, registration, canonical) + if err != nil { + return result, err + } + if matches { + result.Hostnames = []string{canonical} + } + contribution := newDNSObject(dnsContributionGVK, namespace, resourceName) + if err := reader.Get(ctx, client.ObjectKeyFromObject(contribution), contribution); err != nil && !apierrors.IsNotFound(err) { + return result, err + } + if observed && len(result.Hostnames) > 0 && unstructuredConditionCurrent(registration, internalDNSConditionAccepted) && unstructuredConditionCurrent(registration, internalDNSConditionPublished) && unstructuredConditionCurrent(contribution, internalDNSConditionPublished) && attachment.Eligible && attachment.ObservationError == nil { + condition.Status, condition.Reason, condition.Message = metav1.ConditionTrue, internalDNSConditionPublished, "Private name is published" + } + } + if attachment.ObservationError != nil { + condition.Status = metav1.ConditionUnknown + condition.Reason, condition.Message = "NetworkObservationUnavailable", "No current network observation is available" + } else if !attachment.Eligible { + condition.Reason, condition.Message = "AddressUnavailable", "No currently eligible private host address is available" + } + if reconcileErr != nil && attachment.ObservationError == nil { + condition.Reason, condition.Message = "PublicationFailed", "DNS publication could not be reconciled" + } + apimeta.SetStatusCondition(&result.Conditions, condition) + return result, nil +} + +func publicationMatchesContext(ctx context.Context, reader client.Reader, identity InternalDNSProjectIdentity, namespace string, instance *computev1alpha.Instance, attachment instanceDNSAttachment, registration *unstructured.Unstructured, canonical string) (bool, error) { + resolver, err := findResolverContext(ctx, reader, namespace, identity.ProjectUID, attachment.VPCUID) + if err != nil || resolver == nil { + return false, err + } + suffix, zone, err := allocatedDNSNamespace(resolver) + if err != nil { + return false, err + } + registeredZone, _, _ := unstructured.NestedMap(registration.Object, internalDNSSpecField, internalDNSZoneRefField) + return canonical == allocatedInstanceDNSName(instance)+"."+suffix && referenceName(registeredZone) == referenceName(zone), nil +} + +func (r *InternalDNSPublisherReconciler) reportPublication(ctx context.Context, writer client.Client, instance *computev1alpha.Instance, status []computev1alpha.InstanceDNSStatus) error { + if reflect.DeepEqual(instance.Status.DNS, status) { + return nil + } + base := instance.DeepCopy() + instance.Status.DNS = status + if err := writer.Status().Patch(ctx, instance, client.MergeFromWithOptions(base, client.MergeFromWithOptimisticLock{})); err != nil { + return fmt.Errorf("report Instance DNS publication: %w", err) + } + return nil +} diff --git a/internal/controller/internaldns_status_test.go b/internal/controller/internaldns_status_test.go new file mode 100644 index 00000000..8feeacf4 --- /dev/null +++ b/internal/controller/internaldns_status_test.go @@ -0,0 +1,126 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package controller + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/require" + computev1alpha "go.datum.net/compute/api/v1alpha" + networkingv1alpha "go.datum.net/network-services-operator/api/v1alpha" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" +) + +func TestPublicationStatusRequiresCurrentAuthorizedPublication(t *testing.T) { + now := time.Now().UTC().Truncate(time.Second) + instance := &computev1alpha.Instance{ObjectMeta: metav1.ObjectMeta{Name: internalDNSTestInstanceName, UID: internalDNSTestInstanceUID, Generation: 2}} + for _, tc := range []struct { + name string + domain string + foreignZone bool + rejectedRegistration bool + current, observed, wantReady bool + }{ + {name: "published", current: true, observed: true, wantReady: true}, + {name: "custom domain", domain: "private.example.net", current: true, observed: true, wantReady: true}, + {name: "different zone", foreignZone: true, current: true, observed: true}, + {name: "stale generation", observed: true}, + {name: "rejected contribution", current: true}, + {name: "rejected registration", current: true, observed: true, rejectedRegistration: true}, + } { + t.Run(tc.name, func(t *testing.T) { + if tc.domain == "" { + tc.domain = internalDNSTestDefaultDomain + } + s := runtime.NewScheme() + s.AddKnownTypeWithName(dnsResolverContextGVK, &unstructured.Unstructured{}) + s.AddKnownTypeWithName(dnsResolverContextGVK.GroupVersion().WithKind("DNSResolverContextList"), &unstructured.UnstructuredList{}) + resolver := resolverContextObject("context", internalDNSTestProjectUID, internalDNSTestVPCUID, true) + require.NoError(t, unstructured.SetNestedField(resolver.Object, tc.domain, internalDNSStatusField, "managedNamespace", "suffix")) + s.AddKnownTypeWithName(dnsRegistrationGVK, &unstructured.Unstructured{}) + s.AddKnownTypeWithName(dnsContributionGVK, &unstructured.Unstructured{}) + registration := newDNSObject(dnsRegistrationGVK, internalDNSTestNamespace, internalDNSTestInstanceName) + registration.SetGeneration(2) + contribution := newDNSObject(dnsContributionGVK, internalDNSTestNamespace, internalDNSTestInstanceName) + contribution.SetGeneration(2) + observed := int64(1) + if tc.current { + observed = 2 + } + conditions := []any{map[string]any{internalDNSConditionTypeField: internalDNSConditionPublished, internalDNSStatusField: string(metav1.ConditionTrue), internalDNSObservedGenerationField: observed}} + accepted := metav1.ConditionTrue + if tc.rejectedRegistration { + accepted = metav1.ConditionFalse + } + registrationConditions := append(append([]any(nil), conditions...), map[string]any{internalDNSConditionTypeField: internalDNSConditionAccepted, internalDNSStatusField: string(accepted), internalDNSObservedGenerationField: observed}) + registration.Object[internalDNSStatusField] = map[string]any{"canonicalFQDN": allocatedInstanceDNSName(instance) + "." + tc.domain + ".", internalDNSConditionsField: registrationConditions} + zoneName := "zone" + if tc.foreignZone { + zoneName = internalDNSTestForeignValue + } + registration.Object[internalDNSSpecField] = map[string]any{internalDNSZoneRefField: map[string]any{internalDNSNameField: zoneName}} + contribution.Object[internalDNSStatusField] = map[string]any{internalDNSConditionsField: conditions} + cl := fake.NewClientBuilder().WithScheme(s).WithObjects(resolver, registration, contribution).Build() + r := &InternalDNSPublisherReconciler{Now: func() time.Time { return now }} + got, err := r.publicationStatus(context.Background(), cl, InternalDNSProjectIdentity{ProjectUID: internalDNSTestProjectUID}, internalDNSTestNamespace, instance, instanceDNSAttachment{VPCUID: internalDNSTestVPCUID, Eligible: true}, internalDNSTestInstanceName, tc.observed, nil) + require.NoError(t, err) + if tc.foreignZone { + require.Empty(t, got.Hostnames) + } else { + require.Equal(t, []string{allocatedInstanceDNSName(instance) + "." + tc.domain}, got.Hostnames) + } + require.Equal(t, tc.wantReady, got.Conditions[0].Status == metav1.ConditionTrue) + }) + } +} + +func TestReportPublicationPreservesRuntimeStatus(t *testing.T) { + s := runtime.NewScheme() + require.NoError(t, computev1alpha.AddToScheme(s)) + instance := &computev1alpha.Instance{ObjectMeta: metav1.ObjectMeta{Name: internalDNSTestInstanceName, Namespace: internalDNSTestNamespace}, Status: computev1alpha.InstanceStatus{Conditions: []metav1.Condition{{Type: computev1alpha.InstanceReady, Status: metav1.ConditionTrue}}}} + cl := fake.NewClientBuilder().WithScheme(s).WithObjects(instance).WithStatusSubresource(instance).Build() + require.NoError(t, cl.Get(context.Background(), client.ObjectKeyFromObject(instance), instance)) + r := &InternalDNSPublisherReconciler{} + network := networkingv1alpha.NetworkRef{Name: internalDNSTestNetworkName, Namespace: internalDNSTestNamespace} + require.NoError(t, r.reportPublication(context.Background(), cl, instance, []computev1alpha.InstanceDNSStatus{{Network: network}})) + require.NoError(t, cl.Get(context.Background(), client.ObjectKeyFromObject(instance), instance)) + require.Equal(t, metav1.ConditionTrue, instance.Status.Conditions[0].Status) + require.Len(t, instance.Status.DNS, 1) + require.Equal(t, network, instance.Status.DNS[0].Network) +} + +func TestPublicationStatusMatchesNamespacedNetworkReference(t *testing.T) { + network := networkingv1alpha.NetworkRef{Name: internalDNSTestNetworkName, Namespace: internalDNSTestNamespace} + transition := metav1.NewTime(time.Date(2026, time.January, 1, 0, 0, 0, 0, time.UTC)) + for _, tc := range []struct { + name string + network networkingv1alpha.NetworkRef + matches bool + }{ + {name: "same network", network: network, matches: true}, + {name: "different name", network: networkingv1alpha.NetworkRef{Name: "another-network", Namespace: network.Namespace}}, + {name: "same name in another namespace", network: networkingv1alpha.NetworkRef{Name: network.Name, Namespace: "another-namespace"}}, + } { + t.Run(tc.name, func(t *testing.T) { + s := runtime.NewScheme() + s.AddKnownTypeWithName(dnsRegistrationGVK, &unstructured.Unstructured{}) + cl := fake.NewClientBuilder().WithScheme(s).Build() + instance := &computev1alpha.Instance{Status: computev1alpha.InstanceStatus{DNS: []computev1alpha.InstanceDNSStatus{{ + Network: tc.network, + Conditions: []metav1.Condition{{Type: internalDNSConditionReady, Status: metav1.ConditionFalse, Reason: "PendingPublication", LastTransitionTime: transition}}, + }}}} + r := &InternalDNSPublisherReconciler{} + got, err := r.publicationStatus(context.Background(), cl, InternalDNSProjectIdentity{}, internalDNSTestNamespace, instance, instanceDNSAttachment{Network: network, Eligible: true}, internalDNSTestInstanceName, false, nil) + require.NoError(t, err) + require.Equal(t, network, got.Network) + require.Len(t, got.Conditions, 1) + require.Equal(t, tc.matches, got.Conditions[0].LastTransitionTime.Equal(&transition)) + }) + } +} diff --git a/internal/controller/networkinterface_holder.go b/internal/controller/networkinterface_holder.go index d295c382..b5d18ba0 100644 --- a/internal/controller/networkinterface_holder.go +++ b/internal/controller/networkinterface_holder.go @@ -139,11 +139,12 @@ func (r *InstanceReconciler) reconcileHolderAvailability( projected := condition projected.ObservedGeneration = networkInterface.Generation + base := networkInterface.DeepCopy() if !apimeta.SetStatusCondition(&networkInterface.Status.Conditions, projected) { continue } - if err := clusterClient.Status().Update(ctx, &networkInterface); err != nil { + if err := clusterClient.Status().Patch(ctx, &networkInterface, client.MergeFromWithOptions(base, client.MergeFromWithOptimisticLock{})); err != nil { errs = append(errs, fmt.Errorf("update network interface %s status: %w", key, err)) } } diff --git a/internal/controller/networkinterface_holder_test.go b/internal/controller/networkinterface_holder_test.go index 84437cb9..284d4376 100644 --- a/internal/controller/networkinterface_holder_test.go +++ b/internal/controller/networkinterface_holder_test.go @@ -4,15 +4,18 @@ package controller import ( "context" + "encoding/json" "errors" "testing" + jsonpatch "github.com/evanphx/json-patch/v5" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" apimeta "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" computev1alpha "go.datum.net/compute/api/v1alpha" networkingv1alpha "go.datum.net/network-services-operator/api/v1alpha" @@ -162,7 +165,27 @@ func TestReconcileHolderAvailability_BecomesAvailable(t *testing.T) { Status: metav1.ConditionTrue, Reason: computev1alpha.InstanceAvailableReasonAvailable, }) - cl := holderTestClient(instance, newHolderTestInterface()) + iface := newHolderTestInterface() + patched := false + cl := fake.NewClientBuilder().WithScheme(newClaimTestScheme()).WithObjects(instance, iface). + WithStatusSubresource(iface, instance).WithInterceptorFuncs(interceptor.Funcs{ + SubResourcePatch: func(ctx context.Context, cl client.Client, subresource string, object client.Object, patch client.Patch, opts ...client.SubResourcePatchOption) error { + settings := map[string]any{"nameservers": []any{"fd53::53"}, "searches": []any{internalDNSTestDefaultDomain}} + original, err := json.Marshal(map[string]any{"status": map[string]any{"dns": settings}}) + require.NoError(t, err) + data, err := patch.Data(object) + require.NoError(t, err) + merged, err := jsonpatch.MergePatch(original, data) + require.NoError(t, err) + var result map[string]any + require.NoError(t, json.Unmarshal(merged, &result)) + dns := result["status"].(map[string]any)["dns"].(map[string]any) + require.Equal(t, []any{"fd53::53"}, dns["nameservers"]) + require.Equal(t, []any{internalDNSTestDefaultDomain}, dns["searches"]) + patched = true + return cl.SubResource(subresource).Patch(ctx, object, patch, opts...) + }, + }).Build() r := &InstanceReconciler{NetworkingEnabled: true} require.NoError(t, r.reconcileHolderAvailability(context.Background(), cl, instance)) @@ -173,6 +196,7 @@ func TestReconcileHolderAvailability_BecomesAvailable(t *testing.T) { assert.Equal(t, computev1alpha.InstanceAvailableReasonAvailable, condition.Reason) assert.Equal(t, int64(4), condition.ObservedGeneration, "the interface's generation, not the instance's") + require.True(t, patched) } // TestReconcileHolderAvailability_LeavesAvailable covers the drain path: an diff --git a/internal/features/features.go b/internal/features/features.go index 92063f5e..154cad18 100644 --- a/internal/features/features.go +++ b/internal/features/features.go @@ -112,6 +112,9 @@ const ( // // alpha: v0.1 InstanceConsoleSessions featuregate.Feature = "InstanceConsoleSessions" + + // InternalDNSPublishing publishes instance records into project DNS contexts. + InternalDNSPublishing featuregate.Feature = "InternalDNSPublishing" ) // MutableFeatureGate is the mutable feature gate for the compute operator. @@ -132,6 +135,7 @@ func init() { RuntimeClasses: {Default: false, PreRelease: featuregate.Alpha}, InstanceTypes: {Default: false, PreRelease: featuregate.Alpha}, InstanceConsoleSessions: {Default: false, PreRelease: featuregate.Alpha}, + InternalDNSPublishing: {Default: false, PreRelease: featuregate.Alpha}, }); err != nil { panic(err) } diff --git a/internal/features/features_test.go b/internal/features/features_test.go index c1aa5076..b8caa131 100644 --- a/internal/features/features_test.go +++ b/internal/features/features_test.go @@ -135,3 +135,20 @@ func TestInstanceConsoleSessions_ExplicitlyEnabled(t *testing.T) { t.Error("InstanceConsoleSessions = false after Set=true, want true") } } + +func TestInternalDNSPublishing_DefaultDisabled(t *testing.T) { + gate := MutableFeatureGate.DeepCopy() + if gate.Enabled(InternalDNSPublishing) { + t.Error("InternalDNSPublishing default = true, want false") + } +} + +func TestInternalDNSPublishing_ExplicitlyEnabled(t *testing.T) { + gate := MutableFeatureGate.DeepCopy() + if err := gate.Set("InternalDNSPublishing=true"); err != nil { + t.Fatalf("Set(InternalDNSPublishing=true): %v", err) + } + if !gate.Enabled(InternalDNSPublishing) { + t.Error("InternalDNSPublishing = false after Set=true, want true") + } +} diff --git a/test/e2e/referenced-data-multi-location/chainsaw-test.yaml b/test/e2e/referenced-data-multi-location/chainsaw-test.yaml index 4802fce3..9129b696 100644 --- a/test/e2e/referenced-data-multi-location/chainsaw-test.yaml +++ b/test/e2e/referenced-data-multi-location/chainsaw-test.yaml @@ -444,6 +444,10 @@ spec: status: "True" reason: "Ready" + catch: + - script: + content: sh diagnose-referenced-data.sh + - name: assert-instance-ready-on-pop-ord description: | Assert the ord cell did the same. Under the retired delivery path this is @@ -471,6 +475,10 @@ spec: status: "True" reason: "Ready" + catch: + - script: + content: sh diagnose-referenced-data.sh + - name: assert-referenced-data-gate-cleared-on-both-cells description: | Assert the gate itself is gone from both Instances, not just that the diff --git a/test/e2e/referenced-data-multi-location/diagnose-referenced-data.sh b/test/e2e/referenced-data-multi-location/diagnose-referenced-data.sh new file mode 100644 index 00000000..19849d9e --- /dev/null +++ b/test/e2e/referenced-data-multi-location/diagnose-referenced-data.sh @@ -0,0 +1,27 @@ +#!/bin/sh +set -eu + +KDIR=../../../tmp/e2e/kubeconfigs +DOWNSTREAM_NS=$(kubectl --kubeconfig="$KDIR/control-plane.yaml" \ + get namespace "$NAMESPACE" -o template='{{printf "ns-%s" .metadata.uid}}') + +for PLANE in control-plane karmada pop-dfw pop-ord; do + NS="$DOWNSTREAM_NS" + if [ "$PLANE" = control-plane ]; then + NS="$NAMESPACE" + fi + kubectl --kubeconfig="$KDIR/$PLANE.yaml" -n "$NS" \ + get workloaddeployments -o json | jq --arg plane "$PLANE" \ + '.items[] | {plane: $plane, name: .metadata.name, + expected: .metadata.annotations["compute.datumapis.com/expected-referenced-data"], + conditions: .status.conditions}' +done + +for CELL in pop-dfw pop-ord; do + kubectl --kubeconfig="$KDIR/$CELL.yaml" -n "$DOWNSTREAM_NS" \ + get instances -o json | jq --arg cell "$CELL" \ + '.items[] | {cell: $cell, name: .metadata.name, + owner: .metadata.ownerReferences, + gates: .spec.controller.schedulingGates, + conditions: .status.conditions}' +done diff --git a/test/internaldns/README.md b/test/internaldns/README.md new file mode 100644 index 00000000..e684f46c --- /dev/null +++ b/test/internaldns/README.md @@ -0,0 +1,42 @@ +# Internal DNS publisher validation + +The publisher uses the shared Kubernetes internal DNS environment in the +`dns-operator` repository. Use that environment's cluster lifecycle, TLS +admission, NATS transport, and shared serving fleet. Select every API through an +explicit kubeconfig. + +Run the Compute contract tests with Kubernetes envtest assets installed: + +```sh +make envtest +bin/setup-envtest use 1.31.0 --bin-dir bin -p path +go test ./internal/controller ./internal/config ./cmd/... +``` + +The tests cover context identity, flat instance names, atomic endpoint publications, live edge observations, +source lifetime replacement, and DNS-owned publication status. They use fixtures and do not +qualify guest resolution. + +The standalone runner starts the production publisher on an existing project API: + +```sh +go build -o /tmp/compute-dns-publisher ./test/internaldns/publisher +/tmp/compute-dns-publisher --feature-gates=InternalDNSPublishing=true \ + --kubeconfig /tmp/project-publisher.kubeconfig \ + --namespace production --project-uid PROJECT_UID \ + --edge-kubeconfig /tmp/edge-observer.kubeconfig \ + --location dfw +``` + +Provision a resolver context with a `datum.internal` managed zone and scoped +project publisher permissions. Instances must +carry their source lifetime and location labels. Edge namespaces use the project +namespace UID. The observer requires allocated, programmed interfaces, live +claims, Instance-owned runtime Pods, and confirmed node health. +The disabled runner starts without creating Kubernetes clients. + +Release qualification must exercise two VPCs using the same default domain, +real address allocation, guest resolver application, and UDP/TCP queries through +Galactic. Include address changes, withdrawal, publisher outages, stale replay, source replacement, and controller restarts. Run this +qualification in CI and reuse the shared Kubernetes foundation. The publisher +contract tests and runner do not establish that those release criteria pass. diff --git a/test/internaldns/publisher/main.go b/test/internaldns/publisher/main.go new file mode 100644 index 00000000..8377f850 --- /dev/null +++ b/test/internaldns/publisher/main.go @@ -0,0 +1,110 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +// This test runner starts the production publisher against a disposable project +// API without starting unrelated Compute controllers. It is not a product entrypoint. +package main + +import ( + "context" + "flag" + "fmt" + "os" + "time" + + computev1alpha "go.datum.net/compute/api/v1alpha" + "go.datum.net/compute/internal/controller" + "go.datum.net/compute/internal/features" + networkingv1alpha "go.datum.net/network-services-operator/api/v1alpha" + coordinationv1 "k8s.io/api/coordination/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/tools/clientcmd" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/cache" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/cluster" + "sigs.k8s.io/controller-runtime/pkg/log/zap" + metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" + mcmanager "sigs.k8s.io/multicluster-runtime/pkg/manager" + "sigs.k8s.io/multicluster-runtime/pkg/multicluster" + mcsingle "sigs.k8s.io/multicluster-runtime/providers/single" +) + +func main() { + var kubeconfig, namespace, projectUID, gates string + var edgeKubeconfig, location string + flags := flag.NewFlagSet(os.Args[0], flag.ExitOnError) + flags.StringVar(&kubeconfig, "kubeconfig", "", "Disposable project kubeconfig") + flags.StringVar(&namespace, "namespace", "project-e2e", "Project namespace") + flags.StringVar(&projectUID, "project-uid", "project-e2e-uid", "DNS project UID") + flags.StringVar(&gates, "feature-gates", "", "Compute feature gates") + flags.StringVar(&edgeKubeconfig, "edge-kubeconfig", "", "Read-only edge observation kubeconfig") + flags.StringVar(&location, "location", "", "Instance location label") + if err := flags.Parse(os.Args[1:]); err != nil { + panic(err) + } + ctrl.SetLogger(zap.New(zap.UseDevMode(true))) + if err := features.MutableFeatureGate.Set(gates); err != nil { + panic(err) + } + if !features.FeatureGate.Enabled(features.InternalDNSPublishing) { + fmt.Println("InternalDNSPublishing is disabled; no DNS client or controller started") + return + } + if edgeKubeconfig == "" || location == "" { + panic("edge-kubeconfig and location are required") + } + cfg, err := clientcmd.BuildConfigFromFlags("", kubeconfig) + if err != nil { + panic(err) + } + cfg.Timeout = 10 * time.Second + scheme := runtime.NewScheme() + for _, add := range []func(*runtime.Scheme) error{corev1.AddToScheme, coordinationv1.AddToScheme, computev1alpha.AddToScheme, networkingv1alpha.AddToScheme} { + if err := add(scheme); err != nil { + panic(err) + } + } + edgeConfig, err := clientcmd.BuildConfigFromFlags("", edgeKubeconfig) + if err != nil { + panic(err) + } + edgeConfig.Timeout = 5 * time.Second + edge, err := client.New(edgeConfig, client.Options{Scheme: scheme}) + if err != nil { + panic(err) + } + cl, err := cluster.New(cfg, func(o *cluster.Options) { + o.Scheme = scheme + o.Cache.DefaultNamespaces = map[string]cache.Config{namespace: {}} + }) + if err != nil { + panic(err) + } + mgr, err := mcmanager.New(cfg, mcsingle.New(multicluster.ClusterName("single"), cl), ctrl.Options{ + Scheme: scheme, Metrics: metricsserver.Options{BindAddress: "0"}, HealthProbeBindAddress: "0", + Cache: cache.Options{DefaultNamespaces: map[string]cache.Config{namespace: {}}}, + }) + if err != nil { + panic(err) + } + publisher := &controller.InternalDNSPublisherReconciler{ + ObservationSources: map[string]controller.InternalDNSObservationSource{location: {Reader: edge}}, + ProjectIdentities: map[string]controller.InternalDNSProjectIdentity{"single": { + ProjectName: "single", ProjectUID: types.UID(projectUID), + }}, + } + if err := publisher.SetupWithManager(mgr); err != nil { + panic(err) + } + ctx, cancel := context.WithCancel(ctrl.SetupSignalHandler()) + defer cancel() + errs := make(chan error, 2) + go func() { errs <- cl.Start(ctx) }() + go func() { errs <- mgr.Start(ctx) }() + if err := <-errs; err != nil && ctx.Err() == nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } +}