From e091b0cf7e4b8f1ba2ba3ac96718102857490ac8 Mon Sep 17 00:00:00 2001 From: michaelawyu Date: Tue, 21 Oct 2025 22:39:48 +0800 Subject: [PATCH] Progressed drift/diff APIs to v1 Signed-off-by: michaelawyu --- apis/placement/v1/binding_types.go | 36 ++ .../v1/clusterresourceplacement_types.go | 422 ++++++++++++++++- apis/placement/v1/work_types.go | 90 ++++ apis/placement/v1/zz_generated.deepcopy.go | 153 ++++++ ...etes-fleet.io_clusterresourcebindings.yaml | 435 ++++++++++++++++- ...es-fleet.io_clusterresourceplacements.yaml | 439 +++++++++++++++++- ...t.io_clusterresourceplacementstatuses.yaml | 231 +++++++++ .../placement.kubernetes-fleet.io_works.yaml | 342 +++++++++++++- 8 files changed, 2116 insertions(+), 32 deletions(-) diff --git a/apis/placement/v1/binding_types.go b/apis/placement/v1/binding_types.go index 2cd204f81..e11a48536 100644 --- a/apis/placement/v1/binding_types.go +++ b/apis/placement/v1/binding_types.go @@ -105,6 +105,30 @@ type ResourceBindingStatus struct { // +optional FailedPlacements []FailedResourcePlacement `json:"failedPlacements,omitempty"` + // DriftedPlacements is a list of resources that have drifted from their desired states + // kept in the hub cluster, as found by Fleet using the drift detection mechanism. + // + // To control the object size, only the first 100 drifted resources will be included. + // This field is only meaningful if the `ClusterName` is not empty. + // +kubebuilder:validation:Optional + // +kubebuilder:validation:MaxItems=100 + DriftedPlacements []DriftedResourcePlacement `json:"driftedPlacements,omitempty"` + + // DiffedPlacements is a list of resources that have configuration differences from their + // corresponding hub cluster manifests. Fleet will report such differences when: + // + // * The CRP uses the ReportDiff apply strategy, which instructs Fleet to compare the hub + // cluster manifests against the live resources without actually performing any apply op; or + // * Fleet finds a pre-existing resource on the member cluster side that does not match its + // hub cluster counterpart, and the CRP has been configured to only take over a resource if + // no configuration differences are found. + // + // To control the object size, only the first 100 diffed resources will be included. + // This field is only meaningful if the `ClusterName` is not empty. + // +kubebuilder:validation:Optional + // +kubebuilder:validation:MaxItems=100 + DiffedPlacements []DiffedResourcePlacement `json:"diffedPlacements,omitempty"` + // +patchMergeKey=type // +patchStrategy=merge // +listType=map @@ -156,6 +180,18 @@ const ( // - "False" means not all the resources are available in the target cluster yet. // - "Unknown" means we haven't finished the apply yet so that we cannot check the resource availability. ResourceBindingAvailable ResourceBindingConditionType = "Available" + + // ResourceBindingDiffReported indicates that Fleet has successfully reported configuration + // differences between the hub cluster and a specific member cluster for the given resources. + // + // This condition is added only when the ReportDiff apply strategy is used. + // + // It can have the following condition statuses: + // * True: Fleet has successfully reported configuration differences for all resources. + // * False: Fleet has not yet reported configuration differences for some resources, or an + // error has occurred. + // * Unknown: Fleet has not finished processing the diff reporting yet. + ResourceBindingDiffReported ResourceBindingConditionType = "DiffReported" ) // ClusterResourceBindingList is a collection of ClusterResourceBinding. diff --git a/apis/placement/v1/clusterresourceplacement_types.go b/apis/placement/v1/clusterresourceplacement_types.go index aa48ee012..5e0193464 100644 --- a/apis/placement/v1/clusterresourceplacement_types.go +++ b/apis/placement/v1/clusterresourceplacement_types.go @@ -432,27 +432,244 @@ type RolloutStrategy struct { // Note: If multiple CRPs try to place the same resource with different apply strategy, the later ones will fail with the // reason ApplyConflictBetweenPlacements. type ApplyStrategy struct { - // Type defines the type of strategy to use. Default to ClientSideApply. - // Server-side apply is a safer choice. Read more about the differences between server-side apply and client-side - // apply: https://kubernetes.io/docs/reference/using-api/server-side-apply/#comparison-with-client-side-apply. + // ComparisonOption controls how Fleet compares the desired state of a resource, as kept in + // a hub cluster manifest, with the current state of the resource (if applicable) in the + // member cluster. + // + // Available options are: + // + // * PartialComparison: with this option, Fleet will compare only fields that are managed by + // Fleet, i.e., the fields that are specified explicitly in the hub cluster manifest. + // Unmanaged fields are ignored. This is the default option. + // + // * FullComparison: with this option, Fleet will compare all fields of the resource, + // even if the fields are absent from the hub cluster manifest. + // + // Consider using the PartialComparison option if you would like to: + // + // * use the default values for certain fields; or + // * let another agent, e.g., HPAs, VPAs, etc., on the member cluster side manage some fields; or + // * allow ad-hoc or cluster-specific settings on the member cluster side. + // + // To use the FullComparison option, it is recommended that you: + // + // * specify all fields as appropriate in the hub cluster, even if you are OK with using default + // values; + // * make sure that no fields are managed by agents other than Fleet on the member cluster + // side, such as HPAs, VPAs, or other controllers. + // + // See the Fleet documentation for further explanations and usage examples. + // + // +kubebuilder:default=PartialComparison + // +kubebuilder:validation:Enum=PartialComparison;FullComparison + // +kubebuilder:validation:Optional + ComparisonOption ComparisonOptionType `json:"comparisonOption,omitempty"` + + // WhenToApply controls when Fleet would apply the manifests on the hub cluster to the member + // clusters. + // + // Available options are: + // + // * Always: with this option, Fleet will periodically apply hub cluster manifests + // on the member cluster side; this will effectively overwrite any change in the fields + // managed by Fleet (i.e., specified in the hub cluster manifest). This is the default + // option. + // + // Note that this option would revert any ad-hoc changes made on the member cluster side in the + // managed fields; if you would like to make temporary edits on the member cluster side + // in the managed fields, switch to IfNotDrifted option. Note that changes in unmanaged + // fields will be left alone; if you use the FullDiff compare option, such changes will + // be reported as drifts. + // + // * IfNotDrifted: with this option, Fleet will stop applying hub cluster manifests on + // clusters that have drifted from the desired state; apply ops would still continue on + // the rest of the clusters. Drifts are calculated using the ComparisonOption, + // as explained in the corresponding field. + // + // Use this option if you would like Fleet to detect drifts in your multi-cluster setup. + // A drift occurs when an agent makes an ad-hoc change on the member cluster side that + // makes affected resources deviate from its desired state as kept in the hub cluster; + // and this option grants you an opportunity to view the drift details and take actions + // accordingly. The drift details will be reported in the CRP status. + // + // To fix a drift, you may: + // + // * revert the changes manually on the member cluster side + // * update the hub cluster manifest; this will trigger Fleet to apply the latest revision + // of the manifests, which will overwrite the drifted fields + // (if they are managed by Fleet) + // * switch to the Always option; this will trigger Fleet to apply the current revision + // of the manifests, which will overwrite the drifted fields (if they are managed by Fleet). + // * if applicable and necessary, delete the drifted resources on the member cluster side; Fleet + // will attempt to re-create them using the hub cluster manifests + // + // +kubebuilder:default=Always + // +kubebuilder:validation:Enum=Always;IfNotDrifted + // +kubebuilder:validation:Optional + WhenToApply WhenToApplyType `json:"whenToApply,omitempty"` + + // Type is the apply strategy to use; it determines how Fleet applies manifests from the + // hub cluster to a member cluster. + // + // Available options are: + // + // * ClientSideApply: Fleet uses three-way merge to apply manifests, similar to how kubectl + // performs a client-side apply. This is the default option. + // + // Note that this strategy requires that Fleet keep the last applied configuration in the + // annotation of an applied resource. If the object gets so large that apply ops can no longer + // be executed, Fleet will switch to server-side apply. + // + // Use ComparisonOption and WhenToApply settings to control when an apply op can be executed. + // + // * ServerSideApply: Fleet uses server-side apply to apply manifests; Fleet itself will + // become the field manager for specified fields in the manifests. Specify + // ServerSideApplyConfig as appropriate if you would like Fleet to take over field + // ownership upon conflicts. This is the recommended option for most scenarios; it might + // help reduce object size and safely resolve conflicts between field values. For more + // information, please refer to the Kubernetes documentation + // (https://kubernetes.io/docs/reference/using-api/server-side-apply/#comparison-with-client-side-apply). + // + // Use ComparisonOption and WhenToApply settings to control when an apply op can be executed. + // + // * ReportDiff: Fleet will compare the desired state of a resource as kept in the hub cluster + // with its current state (if applicable) on the member cluster side, and report any + // differences. No actual apply ops would be executed, and resources will be left alone as they + // are on the member clusters. + // + // If configuration differences are found on a resource, Fleet will consider this as an apply + // error, which might block rollout depending on the specified rollout strategy. + // + // Use ComparisonOption setting to control how the difference is calculated. + // + // ClientSideApply and ServerSideApply apply strategies only work when Fleet can assume + // ownership of a resource (e.g., the resource is created by Fleet, or Fleet has taken over + // the resource). See the comments on the WhenToTakeOver field for more information. + // ReportDiff apply strategy, however, will function regardless of Fleet's ownership + // status. One may set up a CRP with the ReportDiff strategy and the Never takeover option, + // and this will turn Fleet into a detection tool that reports only configuration differences + // but do not touch any resources on the member cluster side. + // + // For a comparison between the different strategies and usage examples, refer to the + // Fleet documentation. + // // +kubebuilder:default=ClientSideApply - // +kubebuilder:validation:Enum=ClientSideApply;ServerSideApply - // +optional + // +kubebuilder:validation:Enum=ClientSideApply;ServerSideApply;ReportDiff + // +kubebuilder:validation:Optional Type ApplyStrategyType `json:"type,omitempty"` - // AllowCoOwnership defines whether to apply the resource if it already exists in the target cluster and is not - // solely owned by fleet (i.e., metadata.ownerReferences contains only fleet custom resources). - // If true, apply the resource and add fleet as a co-owner. - // If false, leave the resource unchanged and fail the apply. + // AllowCoOwnership controls whether co-ownership between Fleet and other agents are allowed + // on a Fleet-managed resource. If set to false, Fleet will refuse to apply manifests to + // a resource that has been owned by one or more non-Fleet agents. + // + // Note that Fleet does not support the case where one resource is being placed multiple + // times by different CRPs on the same member cluster. An apply error will be returned if + // Fleet finds that a resource has been owned by another placement attempt by Fleet, even + // with the AllowCoOwnership setting set to true. AllowCoOwnership bool `json:"allowCoOwnership,omitempty"` // ServerSideApplyConfig defines the configuration for server side apply. It is honored only when type is ServerSideApply. // +optional ServerSideApplyConfig *ServerSideApplyConfig `json:"serverSideApplyConfig,omitempty"` + + // WhenToTakeOver determines the action to take when Fleet applies resources to a member + // cluster for the first time and finds out that the resource already exists in the cluster. + // + // This setting is most relevant in cases where you would like Fleet to manage pre-existing + // resources on a member cluster. + // + // Available options include: + // + // * Always: with this action, Fleet will apply the hub cluster manifests to the member + // clusters even if the affected resources already exist. This is the default action. + // + // Note that this might lead to fields being overwritten on the member clusters, if they + // are specified in the hub cluster manifests. + // + // * IfNoDiff: with this action, Fleet will apply the hub cluster manifests to the member + // clusters if (and only if) pre-existing resources look the same as the hub cluster manifests. + // + // This is a safer option as pre-existing resources that are inconsistent with the hub cluster + // manifests will not be overwritten; Fleet will ignore them until the inconsistencies + // are resolved properly: any change you make to the hub cluster manifests would not be + // applied, and if you delete the manifests or even the ClusterResourcePlacement itself + // from the hub cluster, these pre-existing resources would not be taken away. + // + // Fleet will check for inconsistencies in accordance with the ComparisonOption setting. See also + // the comments on the ComparisonOption field for more information. + // + // If a diff has been found in a field that is **managed** by Fleet (i.e., the field + // **is specified ** in the hub cluster manifest), consider one of the following actions: + // * set the field in the member cluster to be of the same value as that in the hub cluster + // manifest. + // * update the hub cluster manifest so that its field value matches with that in the member + // cluster. + // * switch to the Always action, which will allow Fleet to overwrite the field with the + // value in the hub cluster manifest. + // + // If a diff has been found in a field that is **not managed** by Fleet (i.e., the field + // **is not specified** in the hub cluster manifest), consider one of the following actions: + // * remove the field from the member cluster. + // * update the hub cluster manifest so that the field is included in the hub cluster manifest. + // + // If appropriate, you may also delete the object from the member cluster; Fleet will recreate + // it using the hub cluster manifest. + // + // * Never: with this action, Fleet will not apply a hub cluster manifest to the member + // clusters if there is a corresponding pre-existing resource. However, if a manifest + // has never been applied yet; or it has a corresponding resource which Fleet has assumed + // ownership, apply op will still be executed. + // + // This is the safest option; one will have to remove the pre-existing resources (so that + // Fleet can re-create them) or switch to a different + // WhenToTakeOver option before Fleet starts processing the corresponding hub cluster + // manifests. + // + // If you prefer Fleet stop processing all manifests, use this option along with the + // ReportDiff apply strategy type. This setup would instruct Fleet to touch nothing + // on the member cluster side but still report configuration differences between the + // hub cluster and member clusters. Fleet will not give up ownership + // that it has already assumed though. + // + // +kubebuilder:default=Always + // +kubebuilder:validation:Enum=Always;IfNoDiff;Never + // +kubebuilder:validation:Optional + WhenToTakeOver WhenToTakeOverType `json:"whenToTakeOver,omitempty"` } -// ApplyStrategyType describes the type of the strategy used to resolve the conflict if the resource to be placed already -// exists in the target cluster and is owned by other appliers. +// ComparisonOptionType describes the compare option that Fleet uses to detect drifts and/or +// calculate differences. +// +enum +type ComparisonOptionType string + +const ( + // ComparisonOptionTypePartialComparison will compare only fields that are managed by Fleet, i.e., + // fields that are specified explicitly in the hub cluster manifest. Unmanaged fields + // are ignored. + ComparisonOptionTypePartialComparison ComparisonOptionType = "PartialComparison" + + // ComparisonOptionTypeFullDiff will compare all fields of the resource, even if the fields + // are absent from the hub cluster manifest. + ComparisonOptionTypeFullComparison ComparisonOptionType = "FullComparison" +) + +// WhenToApplyType describes when Fleet would apply the manifests on the hub cluster to +// the member clusters. +type WhenToApplyType string + +const ( + // WhenToApplyTypeAlways instructs Fleet to periodically apply hub cluster manifests + // on the member cluster side; this will effectively overwrite any change in the fields + // managed by Fleet (i.e., specified in the hub cluster manifest). + WhenToApplyTypeAlways WhenToApplyType = "Always" + + // WhenToApplyTypeIfNotDrifted instructs Fleet to stop applying hub cluster manifests on + // clusters that have drifted from the desired state; apply ops would still continue on + // the rest of the clusters. + WhenToApplyTypeIfNotDrifted WhenToApplyType = "IfNotDrifted" +) + +// ApplyStrategyType describes the type of the strategy used to apply the resource to the target cluster. // +enum type ApplyStrategyType string @@ -466,6 +683,11 @@ const ( // and the existing resource in the target cluster. // Details: https://kubernetes.io/docs/reference/using-api/server-side-apply ApplyStrategyTypeServerSideApply ApplyStrategyType = "ServerSideApply" + + // ApplyStrategyTypeReportDiff will report differences between the desired state of a + // resource as kept in the hub cluster and its current state (if applicable) on the member + // cluster side. No actual apply ops would be executed. + ApplyStrategyTypeReportDiff ApplyStrategyType = "ReportDiff" ) // ServerSideApplyConfig defines the configuration for server side apply. @@ -482,6 +704,43 @@ type ServerSideApplyConfig struct { ForceConflicts bool `json:"force"` } +// WhenToTakeOverType describes the type of the action to take when we first apply the +// resources to the member cluster. +// +enum +type WhenToTakeOverType string + +const ( + // WhenToTakeOverTypeIfNoDiff instructs Fleet to apply a manifest with a corresponding + // pre-existing resource on a member cluster if and only if the pre-existing resource + // looks the same as the manifest. Should there be any inconsistency, Fleet will skip + // the apply op; no change will be made on the resource and Fleet will not claim + // ownership on it. + // + // Note that this will not stop Fleet from processing other manifests in the same + // placement that do not concern the takeover process (e.g., the manifests that have + // not been created yet, or that are already under the management of Fleet). + WhenToTakeOverTypeIfNoDiff WhenToTakeOverType = "IfNoDiff" + + // WhenToTakeOverTypeAlways instructs Fleet to always apply manifests to a member cluster, + // even if there are some corresponding pre-existing resources. Some fields on these + // resources might be overwritten, and Fleet will claim ownership on them. + WhenToTakeOverTypeAlways WhenToTakeOverType = "Always" + + // WhenToTakeOverTypeNever instructs Fleet to never apply a manifest to a member cluster + // if there is a corresponding pre-existing resource. + // + // Note that this will not stop Fleet from processing other manifests in the same placement + // that do not concern the takeover process (e.g., the manifests that have not been created + // yet, or that are already under the management of Fleet). + // + // If you would like Fleet to stop processing manifests all together and do not assume + // ownership on any pre-existing resources, use this option along with the ReportDiff + // apply strategy type. This setup would instruct Fleet to touch nothing on the member + // cluster side but still report configuration differences between the hub cluster + // and member clusters. Fleet will not give up ownership that it has already assumed, though. + WhenToTakeOverTypeNever WhenToTakeOverType = "Never" +) + // +enum type RolloutStrategyType string @@ -657,6 +916,30 @@ type ResourcePlacementStatus struct { // +optional FailedPlacements []FailedResourcePlacement `json:"failedPlacements,omitempty"` + // DriftedPlacements is a list of resources that have drifted from their desired states + // kept in the hub cluster, as found by Fleet using the drift detection mechanism. + // + // To control the object size, only the first 100 drifted resources will be included. + // This field is only meaningful if the `ClusterName` is not empty. + // +kubebuilder:validation:Optional + // +kubebuilder:validation:MaxItems=100 + DriftedPlacements []DriftedResourcePlacement `json:"driftedPlacements,omitempty"` + + // DiffedPlacements is a list of resources that have configuration differences from their + // corresponding hub cluster manifests. Fleet will report such differences when: + // + // * The CRP uses the ReportDiff apply strategy, which instructs Fleet to compare the hub + // cluster manifests against the live resources without actually performing any apply op; or + // * Fleet finds a pre-existing resource on the member cluster side that does not match its + // hub cluster counterpart, and the CRP has been configured to only take over a resource if + // no configuration differences are found. + // + // To control the object size, only the first 100 diffed resources will be included. + // This field is only meaningful if the `ClusterName` is not empty. + // +kubebuilder:validation:Optional + // +kubebuilder:validation:MaxItems=100 + DiffedPlacements []DiffedResourcePlacement `json:"diffedPlacements,omitempty"` + // Conditions is an array of current observed conditions for ResourcePlacementStatus. // +optional Conditions []metav1.Condition `json:"conditions,omitempty"` @@ -672,6 +955,101 @@ type FailedResourcePlacement struct { Condition metav1.Condition `json:"condition"` } +// PatchDetail describes a patch that explains an observed configuration drift or +// difference. +// +// A patch detail can be transcribed as a JSON patch operation, as specified in RFC 6902. +type PatchDetail struct { + // The JSON path that points to a field that has drifted or has configuration differences. + // +kubebuilder:validation:Required + Path string `json:"path"` + + // The value at the JSON path from the member cluster side. + // + // This field can be empty if the JSON path does not exist on the member cluster side; i.e., + // applying the manifest from the hub cluster side would add a new field. + // +kubebuilder:validation:Optional + ValueInMember string `json:"valueInMember,omitempty"` + + // The value at the JSON path from the hub cluster side. + // + // This field can be empty if the JSON path does not exist on the hub cluster side; i.e., + // applying the manifest from the hub cluster side would remove the field. + // +kubebuilder:validation:Optional + ValueInHub string `json:"valueInHub,omitempty"` +} + +// DriftedResourcePlacement contains the details of a resource with configuration drifts. +type DriftedResourcePlacement struct { + // The resource that has drifted. + ResourceIdentifier `json:",inline"` + + // ObservationTime is the time when we observe the configuration drifts for the resource. + // +kubebuilder:validation:Required + // +kubebuilder:validation:Type=string + // +kubebuilder:validation:Format=date-time + ObservationTime metav1.Time `json:"observationTime"` + + // TargetClusterObservedGeneration is the generation of the resource on the target cluster + // that contains the configuration drifts. + // +kubebuilder:validation:Required + TargetClusterObservedGeneration int64 `json:"targetClusterObservedGeneration"` + + // FirstDriftedObservedTime is the first time the resource on the target cluster is + // observed to have configuration drifts. + // +kubebuilder:validation:Required + // +kubebuilder:validation:Type=string + // +kubebuilder:validation:Format=date-time + FirstDriftedObservedTime metav1.Time `json:"firstDriftedObservedTime"` + + // ObservedDrifts are the details about the found configuration drifts. Note that + // Fleet might truncate the details as appropriate to control the object size. + // + // Each detail entry specifies how the live state (the state on the member + // cluster side) compares against the desired state (the state kept in the hub cluster manifest). + // + // An event about the details will be emitted as well. + // +kubebuilder:validation:Optional + ObservedDrifts []PatchDetail `json:"observedDrifts,omitempty"` +} + +// DiffedResourcePlacement contains the details of a resource with configuration differences. +type DiffedResourcePlacement struct { + // The resource that has drifted. + ResourceIdentifier `json:",inline"` + + // ObservationTime is the time when we observe the configuration differences for the resource. + // +kubebuilder:validation:Required + // +kubebuilder:validation:Type=string + // +kubebuilder:validation:Format=date-time + ObservationTime metav1.Time `json:"observationTime"` + + // TargetClusterObservedGeneration is the generation of the resource on the target cluster + // that contains the configuration differences. + // + // This might be nil if the resource has not been created yet on the target cluster. + // + // +kubebuilder:validation:Optional + TargetClusterObservedGeneration *int64 `json:"targetClusterObservedGeneration"` + + // FirstDiffedObservedTime is the first time the resource on the target cluster is + // observed to have configuration differences. + // +kubebuilder:validation:Required + // +kubebuilder:validation:Type=string + // +kubebuilder:validation:Format=date-time + FirstDiffedObservedTime metav1.Time `json:"firstDiffedObservedTime"` + + // ObservedDiffs are the details about the found configuration differences. Note that + // Fleet might truncate the details as appropriate to control the object size. + // + // Each detail entry specifies how the live state (the state on the member + // cluster side) compares against the desired state (the state kept in the hub cluster manifest). + // + // An event about the details will be emitted as well. + // +kubebuilder:validation:Optional + ObservedDiffs []PatchDetail `json:"observedDiffs,omitempty"` +} + // Toleration allows ClusterResourcePlacement to tolerate any taint that matches // the triple using the matching operator . type Toleration struct { @@ -758,6 +1136,17 @@ const ( // array. // - "Unknown" means we haven't finished the apply yet so that we cannot check the resource availability. ClusterResourcePlacementAvailableConditionType ClusterResourcePlacementConditionType = "ClusterResourcePlacementAvailable" + + // ClusterResourcePlacementDiffReportedConditionType indicates whether Fleet has reported + // configuration differences between the desired states of resources as kept in the hub cluster + // and the current states on the all member clusters. + // + // It can have the following condition statuses: + // * True: Fleet has reported complete sets of configuration differences on all member clusters. + // * False: Fleet has not yet reported complete sets of configuration differences on some member + // clusters, or an error has occurred. + // * Unknown: Fleet has not finished processing the diff reporting yet. + ClusterResourcePlacementDiffReportedConditionType ClusterResourcePlacementConditionType = "ClusterResourcePlacementDiffReported" ) // ResourcePlacementConditionType defines a specific condition of a resource placement. @@ -813,6 +1202,17 @@ const ( // - "False" means some of them are not available yet. // - "Unknown" means we haven't finished the apply yet so that we cannot check the resource availability. ResourcesAvailableConditionType ResourcePlacementConditionType = "Available" + + // ResourcePlacementDiffReportedConditionType indicates whether Fleet has reported + // configuration differences between the desired states of resources as kept in the hub cluster + // and the current states on the all member clusters. + // + // It can have the following condition statuses: + // * True: Fleet has reported complete sets of configuration differences on all member clusters. + // * False: Fleet has not yet reported complete sets of configuration differences on some member + // clusters, or an error has occurred. + // * Unknown: Fleet has not finished processing the diff reporting yet. + ResourcePlacementDiffReportedConditionType ResourcePlacementConditionType = "ResourcePlacementDiffReported" ) // PlacementType identifies the type of placement. diff --git a/apis/placement/v1/work_types.go b/apis/placement/v1/work_types.go index 8910ff737..cef2700ea 100644 --- a/apis/placement/v1/work_types.go +++ b/apis/placement/v1/work_types.go @@ -110,6 +110,74 @@ type WorkResourceIdentifier struct { Name string `json:"name,omitempty"` } +// DriftDetails describes the observed configuration drifts. +type DriftDetails struct { + // ObservationTime is the timestamp when the drift was last detected. + // + // +kubebuilder:validation:Required + // +kubebuilder:validation:Type=string + // +kubebuilder:validation:Format=date-time + ObservationTime metav1.Time `json:"observationTime"` + + // ObservedInMemberClusterGeneration is the generation of the applied manifest on the member + // cluster side. + // +kubebuilder:validation:Required + ObservedInMemberClusterGeneration int64 `json:"observedInMemberClusterGeneration"` + + // FirstDriftedObservedTime is the timestamp when the drift was first detected. + // + // +kubebuilder:validation:Required + // +kubebuilder:validation:Type=string + // +kubebuilder:validation:Format=date-time + FirstDriftedObservedTime metav1.Time `json:"firstDriftedObservedTime"` + + // ObservedDrifts describes each drifted field found from the applied manifest. + // Fleet might truncate the details as appropriate to control object size. + // + // Each entry specifies how the live state (the state on the member cluster side) compares + // against the desired state (the state kept in the hub cluster manifest). + // + // +kubebuilder:validation:Optional + ObservedDrifts []PatchDetail `json:"observedDrifts,omitempty"` +} + +// DiffDetails describes the observed configuration differences. +type DiffDetails struct { + // ObservationTime is the timestamp when the configuration difference was last detected. + // + // +kubebuilder:validation:Required + // +kubebuilder:validation:Type=string + // +kubebuilder:validation:Format=date-time + ObservationTime metav1.Time `json:"observationTime"` + + // ObservedInMemberClusterGeneration is the generation of the applied manifest on the member + // cluster side. + // + // This might be nil if the resource has not been created yet in the member cluster. + // + // +kubebuilder:validation:Optional + ObservedInMemberClusterGeneration *int64 `json:"observedInMemberClusterGeneration"` + + // FirstDiffedObservedTime is the timestamp when the configuration difference + // was first detected. + // + // +kubebuilder:validation:Required + // +kubebuilder:validation:Type=string + // +kubebuilder:validation:Format=date-time + FirstDiffedObservedTime metav1.Time `json:"firstDiffedObservedTime"` + + // ObservedDiffs describes each field with configuration difference as found from the + // member cluster side. + // + // Fleet might truncate the details as appropriate to control object size. + // + // Each entry specifies how the live state (the state on the member cluster side) compares + // against the desired state (the state kept in the hub cluster manifest). + // + // +kubebuilder:validation:Optional + ObservedDiffs []PatchDetail `json:"observedDiffs,omitempty"` +} + // ManifestCondition represents the conditions of the resources deployed on // spoke cluster. type ManifestCondition struct { @@ -120,6 +188,28 @@ type ManifestCondition struct { // Conditions represents the conditions of this resource on spoke cluster // +required Conditions []metav1.Condition `json:"conditions"` + + // DriftDetails explains about the observed configuration drifts. + // Fleet might truncate the details as appropriate to control object size. + // + // Note that configuration drifts can only occur on a resource if it is currently owned by + // Fleet and its corresponding placement is set to use the ClientSideApply or ServerSideApply + // apply strategy. In other words, DriftDetails and DiffDetails will not be populated + // at the same time. + // + // +kubebuilder:validation:Optional + DriftDetails *DriftDetails `json:"driftDetails,omitempty"` + + // DiffDetails explains the details about the observed configuration differences. + // Fleet might truncate the details as appropriate to control object size. + // + // Note that configuration differences can only occur on a resource if it is not currently owned + // by Fleet (i.e., it is a pre-existing resource that needs to be taken over), or if its + // corresponding placement is set to use the ReportDiff apply strategy. In other words, + // DiffDetails and DriftDetails will not be populated at the same time. + // + // +kubebuilder:validation:Optional + DiffDetails *DiffDetails `json:"diffDetails,omitempty"` } // +genclient diff --git a/apis/placement/v1/zz_generated.deepcopy.go b/apis/placement/v1/zz_generated.deepcopy.go index 3f10e1e10..fc248dbd2 100644 --- a/apis/placement/v1/zz_generated.deepcopy.go +++ b/apis/placement/v1/zz_generated.deepcopy.go @@ -794,6 +794,106 @@ func (in *ClusterSelectorTerm) DeepCopy() *ClusterSelectorTerm { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *DiffDetails) DeepCopyInto(out *DiffDetails) { + *out = *in + in.ObservationTime.DeepCopyInto(&out.ObservationTime) + if in.ObservedInMemberClusterGeneration != nil { + in, out := &in.ObservedInMemberClusterGeneration, &out.ObservedInMemberClusterGeneration + *out = new(int64) + **out = **in + } + in.FirstDiffedObservedTime.DeepCopyInto(&out.FirstDiffedObservedTime) + if in.ObservedDiffs != nil { + in, out := &in.ObservedDiffs, &out.ObservedDiffs + *out = make([]PatchDetail, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DiffDetails. +func (in *DiffDetails) DeepCopy() *DiffDetails { + if in == nil { + return nil + } + out := new(DiffDetails) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *DiffedResourcePlacement) DeepCopyInto(out *DiffedResourcePlacement) { + *out = *in + in.ResourceIdentifier.DeepCopyInto(&out.ResourceIdentifier) + in.ObservationTime.DeepCopyInto(&out.ObservationTime) + if in.TargetClusterObservedGeneration != nil { + in, out := &in.TargetClusterObservedGeneration, &out.TargetClusterObservedGeneration + *out = new(int64) + **out = **in + } + in.FirstDiffedObservedTime.DeepCopyInto(&out.FirstDiffedObservedTime) + if in.ObservedDiffs != nil { + in, out := &in.ObservedDiffs, &out.ObservedDiffs + *out = make([]PatchDetail, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DiffedResourcePlacement. +func (in *DiffedResourcePlacement) DeepCopy() *DiffedResourcePlacement { + if in == nil { + return nil + } + out := new(DiffedResourcePlacement) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *DriftDetails) DeepCopyInto(out *DriftDetails) { + *out = *in + in.ObservationTime.DeepCopyInto(&out.ObservationTime) + in.FirstDriftedObservedTime.DeepCopyInto(&out.FirstDriftedObservedTime) + if in.ObservedDrifts != nil { + in, out := &in.ObservedDrifts, &out.ObservedDrifts + *out = make([]PatchDetail, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DriftDetails. +func (in *DriftDetails) DeepCopy() *DriftDetails { + if in == nil { + return nil + } + out := new(DriftDetails) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *DriftedResourcePlacement) DeepCopyInto(out *DriftedResourcePlacement) { + *out = *in + in.ResourceIdentifier.DeepCopyInto(&out.ResourceIdentifier) + in.ObservationTime.DeepCopyInto(&out.ObservationTime) + in.FirstDriftedObservedTime.DeepCopyInto(&out.FirstDriftedObservedTime) + if in.ObservedDrifts != nil { + in, out := &in.ObservedDrifts, &out.ObservedDrifts + *out = make([]PatchDetail, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DriftedResourcePlacement. +func (in *DriftedResourcePlacement) DeepCopy() *DriftedResourcePlacement { + if in == nil { + return nil + } + out := new(DriftedResourcePlacement) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *EnvelopeIdentifier) DeepCopyInto(out *EnvelopeIdentifier) { *out = *in @@ -869,6 +969,16 @@ func (in *ManifestCondition) DeepCopyInto(out *ManifestCondition) { (*in)[i].DeepCopyInto(&(*out)[i]) } } + if in.DriftDetails != nil { + in, out := &in.DriftDetails, &out.DriftDetails + *out = new(DriftDetails) + (*in).DeepCopyInto(*out) + } + if in.DiffDetails != nil { + in, out := &in.DiffDetails, &out.DiffDetails + *out = new(DiffDetails) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ManifestCondition. @@ -945,6 +1055,21 @@ func (in *OverrideRule) DeepCopy() *OverrideRule { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PatchDetail) DeepCopyInto(out *PatchDetail) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PatchDetail. +func (in *PatchDetail) DeepCopy() *PatchDetail { + if in == nil { + return nil + } + out := new(PatchDetail) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *PlacementPolicy) DeepCopyInto(out *PlacementPolicy) { *out = *in @@ -1116,6 +1241,20 @@ func (in *ResourceBindingStatus) DeepCopyInto(out *ResourceBindingStatus) { (*in)[i].DeepCopyInto(&(*out)[i]) } } + if in.DriftedPlacements != nil { + in, out := &in.DriftedPlacements, &out.DriftedPlacements + *out = make([]DriftedResourcePlacement, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.DiffedPlacements != nil { + in, out := &in.DiffedPlacements, &out.DiffedPlacements + *out = make([]DiffedResourcePlacement, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } if in.Conditions != nil { in, out := &in.Conditions, &out.Conditions *out = make([]metav1.Condition, len(*in)) @@ -1358,6 +1497,20 @@ func (in *ResourcePlacementStatus) DeepCopyInto(out *ResourcePlacementStatus) { (*in)[i].DeepCopyInto(&(*out)[i]) } } + if in.DriftedPlacements != nil { + in, out := &in.DriftedPlacements, &out.DriftedPlacements + *out = make([]DriftedResourcePlacement, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.DiffedPlacements != nil { + in, out := &in.DiffedPlacements, &out.DiffedPlacements + *out = make([]DiffedResourcePlacement, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } if in.Conditions != nil { in, out := &in.Conditions, &out.Conditions *out = make([]metav1.Condition, len(*in)) diff --git a/config/crd/bases/placement.kubernetes-fleet.io_clusterresourcebindings.yaml b/config/crd/bases/placement.kubernetes-fleet.io_clusterresourcebindings.yaml index 56a8608fb..0e12d6032 100644 --- a/config/crd/bases/placement.kubernetes-fleet.io_clusterresourcebindings.yaml +++ b/config/crd/bases/placement.kubernetes-fleet.io_clusterresourcebindings.yaml @@ -68,11 +68,49 @@ spec: properties: allowCoOwnership: description: |- - AllowCoOwnership defines whether to apply the resource if it already exists in the target cluster and is not - solely owned by fleet (i.e., metadata.ownerReferences contains only fleet custom resources). - If true, apply the resource and add fleet as a co-owner. - If false, leave the resource unchanged and fail the apply. + AllowCoOwnership controls whether co-ownership between Fleet and other agents are allowed + on a Fleet-managed resource. If set to false, Fleet will refuse to apply manifests to + a resource that has been owned by one or more non-Fleet agents. + + Note that Fleet does not support the case where one resource is being placed multiple + times by different CRPs on the same member cluster. An apply error will be returned if + Fleet finds that a resource has been owned by another placement attempt by Fleet, even + with the AllowCoOwnership setting set to true. type: boolean + comparisonOption: + default: PartialComparison + description: |- + ComparisonOption controls how Fleet compares the desired state of a resource, as kept in + a hub cluster manifest, with the current state of the resource (if applicable) in the + member cluster. + + Available options are: + + * PartialComparison: with this option, Fleet will compare only fields that are managed by + Fleet, i.e., the fields that are specified explicitly in the hub cluster manifest. + Unmanaged fields are ignored. This is the default option. + + * FullComparison: with this option, Fleet will compare all fields of the resource, + even if the fields are absent from the hub cluster manifest. + + Consider using the PartialComparison option if you would like to: + + * use the default values for certain fields; or + * let another agent, e.g., HPAs, VPAs, etc., on the member cluster side manage some fields; or + * allow ad-hoc or cluster-specific settings on the member cluster side. + + To use the FullComparison option, it is recommended that you: + + * specify all fields as appropriate in the hub cluster, even if you are OK with using default + values; + * make sure that no fields are managed by agents other than Fleet on the member cluster + side, such as HPAs, VPAs, or other controllers. + + See the Fleet documentation for further explanations and usage examples. + enum: + - PartialComparison + - FullComparison + type: string serverSideApplyConfig: description: ServerSideApplyConfig defines the configuration for server side apply. It is honored only when type is ServerSideApply. @@ -91,12 +129,164 @@ spec: type: default: ClientSideApply description: |- - Type defines the type of strategy to use. Default to ClientSideApply. - Server-side apply is a safer choice. Read more about the differences between server-side apply and client-side - apply: https://kubernetes.io/docs/reference/using-api/server-side-apply/#comparison-with-client-side-apply. + Type is the apply strategy to use; it determines how Fleet applies manifests from the + hub cluster to a member cluster. + + Available options are: + + * ClientSideApply: Fleet uses three-way merge to apply manifests, similar to how kubectl + performs a client-side apply. This is the default option. + + Note that this strategy requires that Fleet keep the last applied configuration in the + annotation of an applied resource. If the object gets so large that apply ops can no longer + be executed, Fleet will switch to server-side apply. + + Use ComparisonOption and WhenToApply settings to control when an apply op can be executed. + + * ServerSideApply: Fleet uses server-side apply to apply manifests; Fleet itself will + become the field manager for specified fields in the manifests. Specify + ServerSideApplyConfig as appropriate if you would like Fleet to take over field + ownership upon conflicts. This is the recommended option for most scenarios; it might + help reduce object size and safely resolve conflicts between field values. For more + information, please refer to the Kubernetes documentation + (https://kubernetes.io/docs/reference/using-api/server-side-apply/#comparison-with-client-side-apply). + + Use ComparisonOption and WhenToApply settings to control when an apply op can be executed. + + * ReportDiff: Fleet will compare the desired state of a resource as kept in the hub cluster + with its current state (if applicable) on the member cluster side, and report any + differences. No actual apply ops would be executed, and resources will be left alone as they + are on the member clusters. + + If configuration differences are found on a resource, Fleet will consider this as an apply + error, which might block rollout depending on the specified rollout strategy. + + Use ComparisonOption setting to control how the difference is calculated. + + ClientSideApply and ServerSideApply apply strategies only work when Fleet can assume + ownership of a resource (e.g., the resource is created by Fleet, or Fleet has taken over + the resource). See the comments on the WhenToTakeOver field for more information. + ReportDiff apply strategy, however, will function regardless of Fleet's ownership + status. One may set up a CRP with the ReportDiff strategy and the Never takeover option, + and this will turn Fleet into a detection tool that reports only configuration differences + but do not touch any resources on the member cluster side. + + For a comparison between the different strategies and usage examples, refer to the + Fleet documentation. enum: - ClientSideApply - ServerSideApply + - ReportDiff + type: string + whenToApply: + default: Always + description: |- + WhenToApply controls when Fleet would apply the manifests on the hub cluster to the member + clusters. + + Available options are: + + * Always: with this option, Fleet will periodically apply hub cluster manifests + on the member cluster side; this will effectively overwrite any change in the fields + managed by Fleet (i.e., specified in the hub cluster manifest). This is the default + option. + + Note that this option would revert any ad-hoc changes made on the member cluster side in the + managed fields; if you would like to make temporary edits on the member cluster side + in the managed fields, switch to IfNotDrifted option. Note that changes in unmanaged + fields will be left alone; if you use the FullDiff compare option, such changes will + be reported as drifts. + + * IfNotDrifted: with this option, Fleet will stop applying hub cluster manifests on + clusters that have drifted from the desired state; apply ops would still continue on + the rest of the clusters. Drifts are calculated using the ComparisonOption, + as explained in the corresponding field. + + Use this option if you would like Fleet to detect drifts in your multi-cluster setup. + A drift occurs when an agent makes an ad-hoc change on the member cluster side that + makes affected resources deviate from its desired state as kept in the hub cluster; + and this option grants you an opportunity to view the drift details and take actions + accordingly. The drift details will be reported in the CRP status. + + To fix a drift, you may: + + * revert the changes manually on the member cluster side + * update the hub cluster manifest; this will trigger Fleet to apply the latest revision + of the manifests, which will overwrite the drifted fields + (if they are managed by Fleet) + * switch to the Always option; this will trigger Fleet to apply the current revision + of the manifests, which will overwrite the drifted fields (if they are managed by Fleet). + * if applicable and necessary, delete the drifted resources on the member cluster side; Fleet + will attempt to re-create them using the hub cluster manifests + enum: + - Always + - IfNotDrifted + type: string + whenToTakeOver: + default: Always + description: |- + WhenToTakeOver determines the action to take when Fleet applies resources to a member + cluster for the first time and finds out that the resource already exists in the cluster. + + This setting is most relevant in cases where you would like Fleet to manage pre-existing + resources on a member cluster. + + Available options include: + + * Always: with this action, Fleet will apply the hub cluster manifests to the member + clusters even if the affected resources already exist. This is the default action. + + Note that this might lead to fields being overwritten on the member clusters, if they + are specified in the hub cluster manifests. + + * IfNoDiff: with this action, Fleet will apply the hub cluster manifests to the member + clusters if (and only if) pre-existing resources look the same as the hub cluster manifests. + + This is a safer option as pre-existing resources that are inconsistent with the hub cluster + manifests will not be overwritten; Fleet will ignore them until the inconsistencies + are resolved properly: any change you make to the hub cluster manifests would not be + applied, and if you delete the manifests or even the ClusterResourcePlacement itself + from the hub cluster, these pre-existing resources would not be taken away. + + Fleet will check for inconsistencies in accordance with the ComparisonOption setting. See also + the comments on the ComparisonOption field for more information. + + If a diff has been found in a field that is **managed** by Fleet (i.e., the field + **is specified ** in the hub cluster manifest), consider one of the following actions: + * set the field in the member cluster to be of the same value as that in the hub cluster + manifest. + * update the hub cluster manifest so that its field value matches with that in the member + cluster. + * switch to the Always action, which will allow Fleet to overwrite the field with the + value in the hub cluster manifest. + + If a diff has been found in a field that is **not managed** by Fleet (i.e., the field + **is not specified** in the hub cluster manifest), consider one of the following actions: + * remove the field from the member cluster. + * update the hub cluster manifest so that the field is included in the hub cluster manifest. + + If appropriate, you may also delete the object from the member cluster; Fleet will recreate + it using the hub cluster manifest. + + * Never: with this action, Fleet will not apply a hub cluster manifest to the member + clusters if there is a corresponding pre-existing resource. However, if a manifest + has never been applied yet; or it has a corresponding resource which Fleet has assumed + ownership, apply op will still be executed. + + This is the safest option; one will have to remove the pre-existing resources (so that + Fleet can re-create them) or switch to a different + WhenToTakeOver option before Fleet starts processing the corresponding hub cluster + manifests. + + If you prefer Fleet stop processing all manifests, use this option along with the + ReportDiff apply strategy type. This setup would instruct Fleet to touch nothing + on the member cluster side but still report configuration differences between the + hub cluster and member clusters. Fleet will not give up ownership + that it has already assumed though. + enum: + - Always + - IfNoDiff + - Never type: string type: object clusterDecision: @@ -257,6 +447,237 @@ spec: x-kubernetes-list-map-keys: - type x-kubernetes-list-type: map + diffedPlacements: + description: |- + DiffedPlacements is a list of resources that have configuration differences from their + corresponding hub cluster manifests. Fleet will report such differences when: + + * The CRP uses the ReportDiff apply strategy, which instructs Fleet to compare the hub + cluster manifests against the live resources without actually performing any apply op; or + * Fleet finds a pre-existing resource on the member cluster side that does not match its + hub cluster counterpart, and the CRP has been configured to only take over a resource if + no configuration differences are found. + + To control the object size, only the first 100 diffed resources will be included. + This field is only meaningful if the `ClusterName` is not empty. + items: + description: DiffedResourcePlacement contains the details of a resource + with configuration differences. + properties: + envelope: + description: Envelope identifies the envelope object that contains + this resource. + properties: + name: + description: Name of the envelope object. + type: string + namespace: + description: Namespace is the namespace of the envelope + object. Empty if the envelope object is cluster scoped. + type: string + type: + default: ConfigMap + description: Type of the envelope object. + enum: + - ConfigMap + type: string + required: + - name + type: object + firstDiffedObservedTime: + description: |- + FirstDiffedObservedTime is the first time the resource on the target cluster is + observed to have configuration differences. + format: date-time + type: string + group: + description: Group is the group name of the selected resource. + type: string + kind: + description: Kind represents the Kind of the selected resources. + type: string + name: + description: Name of the target resource. + type: string + namespace: + description: Namespace is the namespace of the resource. Empty + if the resource is cluster scoped. + type: string + observationTime: + description: ObservationTime is the time when we observe the + configuration differences for the resource. + format: date-time + type: string + observedDiffs: + description: |- + ObservedDiffs are the details about the found configuration differences. Note that + Fleet might truncate the details as appropriate to control the object size. + + Each detail entry specifies how the live state (the state on the member + cluster side) compares against the desired state (the state kept in the hub cluster manifest). + + An event about the details will be emitted as well. + items: + description: |- + PatchDetail describes a patch that explains an observed configuration drift or + difference. + + A patch detail can be transcribed as a JSON patch operation, as specified in RFC 6902. + properties: + path: + description: The JSON path that points to a field that + has drifted or has configuration differences. + type: string + valueInHub: + description: |- + The value at the JSON path from the hub cluster side. + + This field can be empty if the JSON path does not exist on the hub cluster side; i.e., + applying the manifest from the hub cluster side would remove the field. + type: string + valueInMember: + description: |- + The value at the JSON path from the member cluster side. + + This field can be empty if the JSON path does not exist on the member cluster side; i.e., + applying the manifest from the hub cluster side would add a new field. + type: string + required: + - path + type: object + type: array + targetClusterObservedGeneration: + description: |- + TargetClusterObservedGeneration is the generation of the resource on the target cluster + that contains the configuration differences. + + This might be nil if the resource has not been created yet on the target cluster. + format: int64 + type: integer + version: + description: Version is the version of the selected resource. + type: string + required: + - firstDiffedObservedTime + - kind + - name + - observationTime + - version + type: object + maxItems: 100 + type: array + driftedPlacements: + description: |- + DriftedPlacements is a list of resources that have drifted from their desired states + kept in the hub cluster, as found by Fleet using the drift detection mechanism. + + To control the object size, only the first 100 drifted resources will be included. + This field is only meaningful if the `ClusterName` is not empty. + items: + description: DriftedResourcePlacement contains the details of a + resource with configuration drifts. + properties: + envelope: + description: Envelope identifies the envelope object that contains + this resource. + properties: + name: + description: Name of the envelope object. + type: string + namespace: + description: Namespace is the namespace of the envelope + object. Empty if the envelope object is cluster scoped. + type: string + type: + default: ConfigMap + description: Type of the envelope object. + enum: + - ConfigMap + type: string + required: + - name + type: object + firstDriftedObservedTime: + description: |- + FirstDriftedObservedTime is the first time the resource on the target cluster is + observed to have configuration drifts. + format: date-time + type: string + group: + description: Group is the group name of the selected resource. + type: string + kind: + description: Kind represents the Kind of the selected resources. + type: string + name: + description: Name of the target resource. + type: string + namespace: + description: Namespace is the namespace of the resource. Empty + if the resource is cluster scoped. + type: string + observationTime: + description: ObservationTime is the time when we observe the + configuration drifts for the resource. + format: date-time + type: string + observedDrifts: + description: |- + ObservedDrifts are the details about the found configuration drifts. Note that + Fleet might truncate the details as appropriate to control the object size. + + Each detail entry specifies how the live state (the state on the member + cluster side) compares against the desired state (the state kept in the hub cluster manifest). + + An event about the details will be emitted as well. + items: + description: |- + PatchDetail describes a patch that explains an observed configuration drift or + difference. + + A patch detail can be transcribed as a JSON patch operation, as specified in RFC 6902. + properties: + path: + description: The JSON path that points to a field that + has drifted or has configuration differences. + type: string + valueInHub: + description: |- + The value at the JSON path from the hub cluster side. + + This field can be empty if the JSON path does not exist on the hub cluster side; i.e., + applying the manifest from the hub cluster side would remove the field. + type: string + valueInMember: + description: |- + The value at the JSON path from the member cluster side. + + This field can be empty if the JSON path does not exist on the member cluster side; i.e., + applying the manifest from the hub cluster side would add a new field. + type: string + required: + - path + type: object + type: array + targetClusterObservedGeneration: + description: |- + TargetClusterObservedGeneration is the generation of the resource on the target cluster + that contains the configuration drifts. + format: int64 + type: integer + version: + description: Version is the version of the selected resource. + type: string + required: + - firstDriftedObservedTime + - kind + - name + - observationTime + - targetClusterObservedGeneration + - version + type: object + maxItems: 100 + type: array failedPlacements: description: |- FailedPlacements is a list of all the resources failed to be placed to the given cluster or the resource is unavailable. diff --git a/config/crd/bases/placement.kubernetes-fleet.io_clusterresourceplacements.yaml b/config/crd/bases/placement.kubernetes-fleet.io_clusterresourceplacements.yaml index ec0caf947..59df90d72 100644 --- a/config/crd/bases/placement.kubernetes-fleet.io_clusterresourceplacements.yaml +++ b/config/crd/bases/placement.kubernetes-fleet.io_clusterresourceplacements.yaml @@ -630,11 +630,49 @@ spec: properties: allowCoOwnership: description: |- - AllowCoOwnership defines whether to apply the resource if it already exists in the target cluster and is not - solely owned by fleet (i.e., metadata.ownerReferences contains only fleet custom resources). - If true, apply the resource and add fleet as a co-owner. - If false, leave the resource unchanged and fail the apply. + AllowCoOwnership controls whether co-ownership between Fleet and other agents are allowed + on a Fleet-managed resource. If set to false, Fleet will refuse to apply manifests to + a resource that has been owned by one or more non-Fleet agents. + + Note that Fleet does not support the case where one resource is being placed multiple + times by different CRPs on the same member cluster. An apply error will be returned if + Fleet finds that a resource has been owned by another placement attempt by Fleet, even + with the AllowCoOwnership setting set to true. type: boolean + comparisonOption: + default: PartialComparison + description: |- + ComparisonOption controls how Fleet compares the desired state of a resource, as kept in + a hub cluster manifest, with the current state of the resource (if applicable) in the + member cluster. + + Available options are: + + * PartialComparison: with this option, Fleet will compare only fields that are managed by + Fleet, i.e., the fields that are specified explicitly in the hub cluster manifest. + Unmanaged fields are ignored. This is the default option. + + * FullComparison: with this option, Fleet will compare all fields of the resource, + even if the fields are absent from the hub cluster manifest. + + Consider using the PartialComparison option if you would like to: + + * use the default values for certain fields; or + * let another agent, e.g., HPAs, VPAs, etc., on the member cluster side manage some fields; or + * allow ad-hoc or cluster-specific settings on the member cluster side. + + To use the FullComparison option, it is recommended that you: + + * specify all fields as appropriate in the hub cluster, even if you are OK with using default + values; + * make sure that no fields are managed by agents other than Fleet on the member cluster + side, such as HPAs, VPAs, or other controllers. + + See the Fleet documentation for further explanations and usage examples. + enum: + - PartialComparison + - FullComparison + type: string serverSideApplyConfig: description: ServerSideApplyConfig defines the configuration for server side apply. It is honored only when type is ServerSideApply. @@ -653,12 +691,164 @@ spec: type: default: ClientSideApply description: |- - Type defines the type of strategy to use. Default to ClientSideApply. - Server-side apply is a safer choice. Read more about the differences between server-side apply and client-side - apply: https://kubernetes.io/docs/reference/using-api/server-side-apply/#comparison-with-client-side-apply. + Type is the apply strategy to use; it determines how Fleet applies manifests from the + hub cluster to a member cluster. + + Available options are: + + * ClientSideApply: Fleet uses three-way merge to apply manifests, similar to how kubectl + performs a client-side apply. This is the default option. + + Note that this strategy requires that Fleet keep the last applied configuration in the + annotation of an applied resource. If the object gets so large that apply ops can no longer + be executed, Fleet will switch to server-side apply. + + Use ComparisonOption and WhenToApply settings to control when an apply op can be executed. + + * ServerSideApply: Fleet uses server-side apply to apply manifests; Fleet itself will + become the field manager for specified fields in the manifests. Specify + ServerSideApplyConfig as appropriate if you would like Fleet to take over field + ownership upon conflicts. This is the recommended option for most scenarios; it might + help reduce object size and safely resolve conflicts between field values. For more + information, please refer to the Kubernetes documentation + (https://kubernetes.io/docs/reference/using-api/server-side-apply/#comparison-with-client-side-apply). + + Use ComparisonOption and WhenToApply settings to control when an apply op can be executed. + + * ReportDiff: Fleet will compare the desired state of a resource as kept in the hub cluster + with its current state (if applicable) on the member cluster side, and report any + differences. No actual apply ops would be executed, and resources will be left alone as they + are on the member clusters. + + If configuration differences are found on a resource, Fleet will consider this as an apply + error, which might block rollout depending on the specified rollout strategy. + + Use ComparisonOption setting to control how the difference is calculated. + + ClientSideApply and ServerSideApply apply strategies only work when Fleet can assume + ownership of a resource (e.g., the resource is created by Fleet, or Fleet has taken over + the resource). See the comments on the WhenToTakeOver field for more information. + ReportDiff apply strategy, however, will function regardless of Fleet's ownership + status. One may set up a CRP with the ReportDiff strategy and the Never takeover option, + and this will turn Fleet into a detection tool that reports only configuration differences + but do not touch any resources on the member cluster side. + + For a comparison between the different strategies and usage examples, refer to the + Fleet documentation. enum: - ClientSideApply - ServerSideApply + - ReportDiff + type: string + whenToApply: + default: Always + description: |- + WhenToApply controls when Fleet would apply the manifests on the hub cluster to the member + clusters. + + Available options are: + + * Always: with this option, Fleet will periodically apply hub cluster manifests + on the member cluster side; this will effectively overwrite any change in the fields + managed by Fleet (i.e., specified in the hub cluster manifest). This is the default + option. + + Note that this option would revert any ad-hoc changes made on the member cluster side in the + managed fields; if you would like to make temporary edits on the member cluster side + in the managed fields, switch to IfNotDrifted option. Note that changes in unmanaged + fields will be left alone; if you use the FullDiff compare option, such changes will + be reported as drifts. + + * IfNotDrifted: with this option, Fleet will stop applying hub cluster manifests on + clusters that have drifted from the desired state; apply ops would still continue on + the rest of the clusters. Drifts are calculated using the ComparisonOption, + as explained in the corresponding field. + + Use this option if you would like Fleet to detect drifts in your multi-cluster setup. + A drift occurs when an agent makes an ad-hoc change on the member cluster side that + makes affected resources deviate from its desired state as kept in the hub cluster; + and this option grants you an opportunity to view the drift details and take actions + accordingly. The drift details will be reported in the CRP status. + + To fix a drift, you may: + + * revert the changes manually on the member cluster side + * update the hub cluster manifest; this will trigger Fleet to apply the latest revision + of the manifests, which will overwrite the drifted fields + (if they are managed by Fleet) + * switch to the Always option; this will trigger Fleet to apply the current revision + of the manifests, which will overwrite the drifted fields (if they are managed by Fleet). + * if applicable and necessary, delete the drifted resources on the member cluster side; Fleet + will attempt to re-create them using the hub cluster manifests + enum: + - Always + - IfNotDrifted + type: string + whenToTakeOver: + default: Always + description: |- + WhenToTakeOver determines the action to take when Fleet applies resources to a member + cluster for the first time and finds out that the resource already exists in the cluster. + + This setting is most relevant in cases where you would like Fleet to manage pre-existing + resources on a member cluster. + + Available options include: + + * Always: with this action, Fleet will apply the hub cluster manifests to the member + clusters even if the affected resources already exist. This is the default action. + + Note that this might lead to fields being overwritten on the member clusters, if they + are specified in the hub cluster manifests. + + * IfNoDiff: with this action, Fleet will apply the hub cluster manifests to the member + clusters if (and only if) pre-existing resources look the same as the hub cluster manifests. + + This is a safer option as pre-existing resources that are inconsistent with the hub cluster + manifests will not be overwritten; Fleet will ignore them until the inconsistencies + are resolved properly: any change you make to the hub cluster manifests would not be + applied, and if you delete the manifests or even the ClusterResourcePlacement itself + from the hub cluster, these pre-existing resources would not be taken away. + + Fleet will check for inconsistencies in accordance with the ComparisonOption setting. See also + the comments on the ComparisonOption field for more information. + + If a diff has been found in a field that is **managed** by Fleet (i.e., the field + **is specified ** in the hub cluster manifest), consider one of the following actions: + * set the field in the member cluster to be of the same value as that in the hub cluster + manifest. + * update the hub cluster manifest so that its field value matches with that in the member + cluster. + * switch to the Always action, which will allow Fleet to overwrite the field with the + value in the hub cluster manifest. + + If a diff has been found in a field that is **not managed** by Fleet (i.e., the field + **is not specified** in the hub cluster manifest), consider one of the following actions: + * remove the field from the member cluster. + * update the hub cluster manifest so that the field is included in the hub cluster manifest. + + If appropriate, you may also delete the object from the member cluster; Fleet will recreate + it using the hub cluster manifest. + + * Never: with this action, Fleet will not apply a hub cluster manifest to the member + clusters if there is a corresponding pre-existing resource. However, if a manifest + has never been applied yet; or it has a corresponding resource which Fleet has assumed + ownership, apply op will still be executed. + + This is the safest option; one will have to remove the pre-existing resources (so that + Fleet can re-create them) or switch to a different + WhenToTakeOver option before Fleet starts processing the corresponding hub cluster + manifests. + + If you prefer Fleet stop processing all manifests, use this option along with the + ReportDiff apply strategy type. This setup would instruct Fleet to touch nothing + on the member cluster side but still report configuration differences between the + hub cluster and member clusters. Fleet will not give up ownership + that it has already assumed though. + enum: + - Always + - IfNoDiff + - Never type: string type: object rollingUpdate: @@ -908,6 +1098,241 @@ spec: - type type: object type: array + diffedPlacements: + description: |- + DiffedPlacements is a list of resources that have configuration differences from their + corresponding hub cluster manifests. Fleet will report such differences when: + + * The CRP uses the ReportDiff apply strategy, which instructs Fleet to compare the hub + cluster manifests against the live resources without actually performing any apply op; or + * Fleet finds a pre-existing resource on the member cluster side that does not match its + hub cluster counterpart, and the CRP has been configured to only take over a resource if + no configuration differences are found. + + To control the object size, only the first 100 diffed resources will be included. + This field is only meaningful if the `ClusterName` is not empty. + items: + description: DiffedResourcePlacement contains the details + of a resource with configuration differences. + properties: + envelope: + description: Envelope identifies the envelope object that + contains this resource. + properties: + name: + description: Name of the envelope object. + type: string + namespace: + description: Namespace is the namespace of the envelope + object. Empty if the envelope object is cluster + scoped. + type: string + type: + default: ConfigMap + description: Type of the envelope object. + enum: + - ConfigMap + type: string + required: + - name + type: object + firstDiffedObservedTime: + description: |- + FirstDiffedObservedTime is the first time the resource on the target cluster is + observed to have configuration differences. + format: date-time + type: string + group: + description: Group is the group name of the selected resource. + type: string + kind: + description: Kind represents the Kind of the selected + resources. + type: string + name: + description: Name of the target resource. + type: string + namespace: + description: Namespace is the namespace of the resource. + Empty if the resource is cluster scoped. + type: string + observationTime: + description: ObservationTime is the time when we observe + the configuration differences for the resource. + format: date-time + type: string + observedDiffs: + description: |- + ObservedDiffs are the details about the found configuration differences. Note that + Fleet might truncate the details as appropriate to control the object size. + + Each detail entry specifies how the live state (the state on the member + cluster side) compares against the desired state (the state kept in the hub cluster manifest). + + An event about the details will be emitted as well. + items: + description: |- + PatchDetail describes a patch that explains an observed configuration drift or + difference. + + A patch detail can be transcribed as a JSON patch operation, as specified in RFC 6902. + properties: + path: + description: The JSON path that points to a field + that has drifted or has configuration differences. + type: string + valueInHub: + description: |- + The value at the JSON path from the hub cluster side. + + This field can be empty if the JSON path does not exist on the hub cluster side; i.e., + applying the manifest from the hub cluster side would remove the field. + type: string + valueInMember: + description: |- + The value at the JSON path from the member cluster side. + + This field can be empty if the JSON path does not exist on the member cluster side; i.e., + applying the manifest from the hub cluster side would add a new field. + type: string + required: + - path + type: object + type: array + targetClusterObservedGeneration: + description: |- + TargetClusterObservedGeneration is the generation of the resource on the target cluster + that contains the configuration differences. + + This might be nil if the resource has not been created yet on the target cluster. + format: int64 + type: integer + version: + description: Version is the version of the selected resource. + type: string + required: + - firstDiffedObservedTime + - kind + - name + - observationTime + - version + type: object + maxItems: 100 + type: array + driftedPlacements: + description: |- + DriftedPlacements is a list of resources that have drifted from their desired states + kept in the hub cluster, as found by Fleet using the drift detection mechanism. + + To control the object size, only the first 100 drifted resources will be included. + This field is only meaningful if the `ClusterName` is not empty. + items: + description: DriftedResourcePlacement contains the details + of a resource with configuration drifts. + properties: + envelope: + description: Envelope identifies the envelope object that + contains this resource. + properties: + name: + description: Name of the envelope object. + type: string + namespace: + description: Namespace is the namespace of the envelope + object. Empty if the envelope object is cluster + scoped. + type: string + type: + default: ConfigMap + description: Type of the envelope object. + enum: + - ConfigMap + type: string + required: + - name + type: object + firstDriftedObservedTime: + description: |- + FirstDriftedObservedTime is the first time the resource on the target cluster is + observed to have configuration drifts. + format: date-time + type: string + group: + description: Group is the group name of the selected resource. + type: string + kind: + description: Kind represents the Kind of the selected + resources. + type: string + name: + description: Name of the target resource. + type: string + namespace: + description: Namespace is the namespace of the resource. + Empty if the resource is cluster scoped. + type: string + observationTime: + description: ObservationTime is the time when we observe + the configuration drifts for the resource. + format: date-time + type: string + observedDrifts: + description: |- + ObservedDrifts are the details about the found configuration drifts. Note that + Fleet might truncate the details as appropriate to control the object size. + + Each detail entry specifies how the live state (the state on the member + cluster side) compares against the desired state (the state kept in the hub cluster manifest). + + An event about the details will be emitted as well. + items: + description: |- + PatchDetail describes a patch that explains an observed configuration drift or + difference. + + A patch detail can be transcribed as a JSON patch operation, as specified in RFC 6902. + properties: + path: + description: The JSON path that points to a field + that has drifted or has configuration differences. + type: string + valueInHub: + description: |- + The value at the JSON path from the hub cluster side. + + This field can be empty if the JSON path does not exist on the hub cluster side; i.e., + applying the manifest from the hub cluster side would remove the field. + type: string + valueInMember: + description: |- + The value at the JSON path from the member cluster side. + + This field can be empty if the JSON path does not exist on the member cluster side; i.e., + applying the manifest from the hub cluster side would add a new field. + type: string + required: + - path + type: object + type: array + targetClusterObservedGeneration: + description: |- + TargetClusterObservedGeneration is the generation of the resource on the target cluster + that contains the configuration drifts. + format: int64 + type: integer + version: + description: Version is the version of the selected resource. + type: string + required: + - firstDriftedObservedTime + - kind + - name + - observationTime + - targetClusterObservedGeneration + - version + type: object + maxItems: 100 + type: array failedPlacements: description: |- FailedPlacements is a list of all the resources failed to be placed to the given cluster or the resource is unavailable. diff --git a/config/crd/bases/placement.kubernetes-fleet.io_clusterresourceplacementstatuses.yaml b/config/crd/bases/placement.kubernetes-fleet.io_clusterresourceplacementstatuses.yaml index a474e0244..2c42689db 100644 --- a/config/crd/bases/placement.kubernetes-fleet.io_clusterresourceplacementstatuses.yaml +++ b/config/crd/bases/placement.kubernetes-fleet.io_clusterresourceplacementstatuses.yaml @@ -204,6 +204,237 @@ spec: - type type: object type: array + diffedPlacements: + description: |- + DiffedPlacements is a list of resources that have configuration differences from their + corresponding hub cluster manifests. Fleet will report such differences when: + + * The CRP uses the ReportDiff apply strategy, which instructs Fleet to compare the hub + cluster manifests against the live resources without actually performing any apply op; or + * Fleet finds a pre-existing resource on the member cluster side that does not match its + hub cluster counterpart, and the CRP has been configured to only take over a resource if + no configuration differences are found. + + To control the object size, only the first 100 diffed resources will be included. + This field is only meaningful if the `ClusterName` is not empty. + items: + description: DiffedResourcePlacement contains the details of a + resource with configuration differences. + properties: + envelope: + description: Envelope identifies the envelope object that + contains this resource. + properties: + name: + description: Name of the envelope object. + type: string + namespace: + description: Namespace is the namespace of the envelope + object. Empty if the envelope object is cluster scoped. + type: string + type: + default: ConfigMap + description: Type of the envelope object. + enum: + - ConfigMap + type: string + required: + - name + type: object + firstDiffedObservedTime: + description: |- + FirstDiffedObservedTime is the first time the resource on the target cluster is + observed to have configuration differences. + format: date-time + type: string + group: + description: Group is the group name of the selected resource. + type: string + kind: + description: Kind represents the Kind of the selected resources. + type: string + name: + description: Name of the target resource. + type: string + namespace: + description: Namespace is the namespace of the resource. Empty + if the resource is cluster scoped. + type: string + observationTime: + description: ObservationTime is the time when we observe the + configuration differences for the resource. + format: date-time + type: string + observedDiffs: + description: |- + ObservedDiffs are the details about the found configuration differences. Note that + Fleet might truncate the details as appropriate to control the object size. + + Each detail entry specifies how the live state (the state on the member + cluster side) compares against the desired state (the state kept in the hub cluster manifest). + + An event about the details will be emitted as well. + items: + description: |- + PatchDetail describes a patch that explains an observed configuration drift or + difference. + + A patch detail can be transcribed as a JSON patch operation, as specified in RFC 6902. + properties: + path: + description: The JSON path that points to a field that + has drifted or has configuration differences. + type: string + valueInHub: + description: |- + The value at the JSON path from the hub cluster side. + + This field can be empty if the JSON path does not exist on the hub cluster side; i.e., + applying the manifest from the hub cluster side would remove the field. + type: string + valueInMember: + description: |- + The value at the JSON path from the member cluster side. + + This field can be empty if the JSON path does not exist on the member cluster side; i.e., + applying the manifest from the hub cluster side would add a new field. + type: string + required: + - path + type: object + type: array + targetClusterObservedGeneration: + description: |- + TargetClusterObservedGeneration is the generation of the resource on the target cluster + that contains the configuration differences. + + This might be nil if the resource has not been created yet on the target cluster. + format: int64 + type: integer + version: + description: Version is the version of the selected resource. + type: string + required: + - firstDiffedObservedTime + - kind + - name + - observationTime + - version + type: object + maxItems: 100 + type: array + driftedPlacements: + description: |- + DriftedPlacements is a list of resources that have drifted from their desired states + kept in the hub cluster, as found by Fleet using the drift detection mechanism. + + To control the object size, only the first 100 drifted resources will be included. + This field is only meaningful if the `ClusterName` is not empty. + items: + description: DriftedResourcePlacement contains the details of + a resource with configuration drifts. + properties: + envelope: + description: Envelope identifies the envelope object that + contains this resource. + properties: + name: + description: Name of the envelope object. + type: string + namespace: + description: Namespace is the namespace of the envelope + object. Empty if the envelope object is cluster scoped. + type: string + type: + default: ConfigMap + description: Type of the envelope object. + enum: + - ConfigMap + type: string + required: + - name + type: object + firstDriftedObservedTime: + description: |- + FirstDriftedObservedTime is the first time the resource on the target cluster is + observed to have configuration drifts. + format: date-time + type: string + group: + description: Group is the group name of the selected resource. + type: string + kind: + description: Kind represents the Kind of the selected resources. + type: string + name: + description: Name of the target resource. + type: string + namespace: + description: Namespace is the namespace of the resource. Empty + if the resource is cluster scoped. + type: string + observationTime: + description: ObservationTime is the time when we observe the + configuration drifts for the resource. + format: date-time + type: string + observedDrifts: + description: |- + ObservedDrifts are the details about the found configuration drifts. Note that + Fleet might truncate the details as appropriate to control the object size. + + Each detail entry specifies how the live state (the state on the member + cluster side) compares against the desired state (the state kept in the hub cluster manifest). + + An event about the details will be emitted as well. + items: + description: |- + PatchDetail describes a patch that explains an observed configuration drift or + difference. + + A patch detail can be transcribed as a JSON patch operation, as specified in RFC 6902. + properties: + path: + description: The JSON path that points to a field that + has drifted or has configuration differences. + type: string + valueInHub: + description: |- + The value at the JSON path from the hub cluster side. + + This field can be empty if the JSON path does not exist on the hub cluster side; i.e., + applying the manifest from the hub cluster side would remove the field. + type: string + valueInMember: + description: |- + The value at the JSON path from the member cluster side. + + This field can be empty if the JSON path does not exist on the member cluster side; i.e., + applying the manifest from the hub cluster side would add a new field. + type: string + required: + - path + type: object + type: array + targetClusterObservedGeneration: + description: |- + TargetClusterObservedGeneration is the generation of the resource on the target cluster + that contains the configuration drifts. + format: int64 + type: integer + version: + description: Version is the version of the selected resource. + type: string + required: + - firstDriftedObservedTime + - kind + - name + - observationTime + - targetClusterObservedGeneration + - version + type: object + maxItems: 100 + type: array failedPlacements: description: |- FailedPlacements is a list of all the resources failed to be placed to the given cluster or the resource is unavailable. diff --git a/config/crd/bases/placement.kubernetes-fleet.io_works.yaml b/config/crd/bases/placement.kubernetes-fleet.io_works.yaml index e2220df73..9d6bcfb3b 100644 --- a/config/crd/bases/placement.kubernetes-fleet.io_works.yaml +++ b/config/crd/bases/placement.kubernetes-fleet.io_works.yaml @@ -50,11 +50,49 @@ spec: properties: allowCoOwnership: description: |- - AllowCoOwnership defines whether to apply the resource if it already exists in the target cluster and is not - solely owned by fleet (i.e., metadata.ownerReferences contains only fleet custom resources). - If true, apply the resource and add fleet as a co-owner. - If false, leave the resource unchanged and fail the apply. + AllowCoOwnership controls whether co-ownership between Fleet and other agents are allowed + on a Fleet-managed resource. If set to false, Fleet will refuse to apply manifests to + a resource that has been owned by one or more non-Fleet agents. + + Note that Fleet does not support the case where one resource is being placed multiple + times by different CRPs on the same member cluster. An apply error will be returned if + Fleet finds that a resource has been owned by another placement attempt by Fleet, even + with the AllowCoOwnership setting set to true. type: boolean + comparisonOption: + default: PartialComparison + description: |- + ComparisonOption controls how Fleet compares the desired state of a resource, as kept in + a hub cluster manifest, with the current state of the resource (if applicable) in the + member cluster. + + Available options are: + + * PartialComparison: with this option, Fleet will compare only fields that are managed by + Fleet, i.e., the fields that are specified explicitly in the hub cluster manifest. + Unmanaged fields are ignored. This is the default option. + + * FullComparison: with this option, Fleet will compare all fields of the resource, + even if the fields are absent from the hub cluster manifest. + + Consider using the PartialComparison option if you would like to: + + * use the default values for certain fields; or + * let another agent, e.g., HPAs, VPAs, etc., on the member cluster side manage some fields; or + * allow ad-hoc or cluster-specific settings on the member cluster side. + + To use the FullComparison option, it is recommended that you: + + * specify all fields as appropriate in the hub cluster, even if you are OK with using default + values; + * make sure that no fields are managed by agents other than Fleet on the member cluster + side, such as HPAs, VPAs, or other controllers. + + See the Fleet documentation for further explanations and usage examples. + enum: + - PartialComparison + - FullComparison + type: string serverSideApplyConfig: description: ServerSideApplyConfig defines the configuration for server side apply. It is honored only when type is ServerSideApply. @@ -73,12 +111,164 @@ spec: type: default: ClientSideApply description: |- - Type defines the type of strategy to use. Default to ClientSideApply. - Server-side apply is a safer choice. Read more about the differences between server-side apply and client-side - apply: https://kubernetes.io/docs/reference/using-api/server-side-apply/#comparison-with-client-side-apply. + Type is the apply strategy to use; it determines how Fleet applies manifests from the + hub cluster to a member cluster. + + Available options are: + + * ClientSideApply: Fleet uses three-way merge to apply manifests, similar to how kubectl + performs a client-side apply. This is the default option. + + Note that this strategy requires that Fleet keep the last applied configuration in the + annotation of an applied resource. If the object gets so large that apply ops can no longer + be executed, Fleet will switch to server-side apply. + + Use ComparisonOption and WhenToApply settings to control when an apply op can be executed. + + * ServerSideApply: Fleet uses server-side apply to apply manifests; Fleet itself will + become the field manager for specified fields in the manifests. Specify + ServerSideApplyConfig as appropriate if you would like Fleet to take over field + ownership upon conflicts. This is the recommended option for most scenarios; it might + help reduce object size and safely resolve conflicts between field values. For more + information, please refer to the Kubernetes documentation + (https://kubernetes.io/docs/reference/using-api/server-side-apply/#comparison-with-client-side-apply). + + Use ComparisonOption and WhenToApply settings to control when an apply op can be executed. + + * ReportDiff: Fleet will compare the desired state of a resource as kept in the hub cluster + with its current state (if applicable) on the member cluster side, and report any + differences. No actual apply ops would be executed, and resources will be left alone as they + are on the member clusters. + + If configuration differences are found on a resource, Fleet will consider this as an apply + error, which might block rollout depending on the specified rollout strategy. + + Use ComparisonOption setting to control how the difference is calculated. + + ClientSideApply and ServerSideApply apply strategies only work when Fleet can assume + ownership of a resource (e.g., the resource is created by Fleet, or Fleet has taken over + the resource). See the comments on the WhenToTakeOver field for more information. + ReportDiff apply strategy, however, will function regardless of Fleet's ownership + status. One may set up a CRP with the ReportDiff strategy and the Never takeover option, + and this will turn Fleet into a detection tool that reports only configuration differences + but do not touch any resources on the member cluster side. + + For a comparison between the different strategies and usage examples, refer to the + Fleet documentation. enum: - ClientSideApply - ServerSideApply + - ReportDiff + type: string + whenToApply: + default: Always + description: |- + WhenToApply controls when Fleet would apply the manifests on the hub cluster to the member + clusters. + + Available options are: + + * Always: with this option, Fleet will periodically apply hub cluster manifests + on the member cluster side; this will effectively overwrite any change in the fields + managed by Fleet (i.e., specified in the hub cluster manifest). This is the default + option. + + Note that this option would revert any ad-hoc changes made on the member cluster side in the + managed fields; if you would like to make temporary edits on the member cluster side + in the managed fields, switch to IfNotDrifted option. Note that changes in unmanaged + fields will be left alone; if you use the FullDiff compare option, such changes will + be reported as drifts. + + * IfNotDrifted: with this option, Fleet will stop applying hub cluster manifests on + clusters that have drifted from the desired state; apply ops would still continue on + the rest of the clusters. Drifts are calculated using the ComparisonOption, + as explained in the corresponding field. + + Use this option if you would like Fleet to detect drifts in your multi-cluster setup. + A drift occurs when an agent makes an ad-hoc change on the member cluster side that + makes affected resources deviate from its desired state as kept in the hub cluster; + and this option grants you an opportunity to view the drift details and take actions + accordingly. The drift details will be reported in the CRP status. + + To fix a drift, you may: + + * revert the changes manually on the member cluster side + * update the hub cluster manifest; this will trigger Fleet to apply the latest revision + of the manifests, which will overwrite the drifted fields + (if they are managed by Fleet) + * switch to the Always option; this will trigger Fleet to apply the current revision + of the manifests, which will overwrite the drifted fields (if they are managed by Fleet). + * if applicable and necessary, delete the drifted resources on the member cluster side; Fleet + will attempt to re-create them using the hub cluster manifests + enum: + - Always + - IfNotDrifted + type: string + whenToTakeOver: + default: Always + description: |- + WhenToTakeOver determines the action to take when Fleet applies resources to a member + cluster for the first time and finds out that the resource already exists in the cluster. + + This setting is most relevant in cases where you would like Fleet to manage pre-existing + resources on a member cluster. + + Available options include: + + * Always: with this action, Fleet will apply the hub cluster manifests to the member + clusters even if the affected resources already exist. This is the default action. + + Note that this might lead to fields being overwritten on the member clusters, if they + are specified in the hub cluster manifests. + + * IfNoDiff: with this action, Fleet will apply the hub cluster manifests to the member + clusters if (and only if) pre-existing resources look the same as the hub cluster manifests. + + This is a safer option as pre-existing resources that are inconsistent with the hub cluster + manifests will not be overwritten; Fleet will ignore them until the inconsistencies + are resolved properly: any change you make to the hub cluster manifests would not be + applied, and if you delete the manifests or even the ClusterResourcePlacement itself + from the hub cluster, these pre-existing resources would not be taken away. + + Fleet will check for inconsistencies in accordance with the ComparisonOption setting. See also + the comments on the ComparisonOption field for more information. + + If a diff has been found in a field that is **managed** by Fleet (i.e., the field + **is specified ** in the hub cluster manifest), consider one of the following actions: + * set the field in the member cluster to be of the same value as that in the hub cluster + manifest. + * update the hub cluster manifest so that its field value matches with that in the member + cluster. + * switch to the Always action, which will allow Fleet to overwrite the field with the + value in the hub cluster manifest. + + If a diff has been found in a field that is **not managed** by Fleet (i.e., the field + **is not specified** in the hub cluster manifest), consider one of the following actions: + * remove the field from the member cluster. + * update the hub cluster manifest so that the field is included in the hub cluster manifest. + + If appropriate, you may also delete the object from the member cluster; Fleet will recreate + it using the hub cluster manifest. + + * Never: with this action, Fleet will not apply a hub cluster manifest to the member + clusters if there is a corresponding pre-existing resource. However, if a manifest + has never been applied yet; or it has a corresponding resource which Fleet has assumed + ownership, apply op will still be executed. + + This is the safest option; one will have to remove the pre-existing resources (so that + Fleet can re-create them) or switch to a different + WhenToTakeOver option before Fleet starts processing the corresponding hub cluster + manifests. + + If you prefer Fleet stop processing all manifests, use this option along with the + ReportDiff apply strategy type. This setup would instruct Fleet to touch nothing + on the member cluster side but still report configuration differences between the + hub cluster and member clusters. Fleet will not give up ownership + that it has already assumed though. + enum: + - Always + - IfNoDiff + - Never type: string type: object workload: @@ -233,6 +423,144 @@ spec: - type type: object type: array + diffDetails: + description: |- + DiffDetails explains the details about the observed configuration differences. + Fleet might truncate the details as appropriate to control object size. + + Note that configuration differences can only occur on a resource if it is not currently owned + by Fleet (i.e., it is a pre-existing resource that needs to be taken over), or if its + corresponding placement is set to use the ReportDiff apply strategy. In other words, + DiffDetails and DriftDetails will not be populated at the same time. + properties: + firstDiffedObservedTime: + description: |- + FirstDiffedObservedTime is the timestamp when the configuration difference + was first detected. + format: date-time + type: string + observationTime: + description: ObservationTime is the timestamp when the configuration + difference was last detected. + format: date-time + type: string + observedDiffs: + description: |- + ObservedDiffs describes each field with configuration difference as found from the + member cluster side. + + Fleet might truncate the details as appropriate to control object size. + + Each entry specifies how the live state (the state on the member cluster side) compares + against the desired state (the state kept in the hub cluster manifest). + items: + description: |- + PatchDetail describes a patch that explains an observed configuration drift or + difference. + + A patch detail can be transcribed as a JSON patch operation, as specified in RFC 6902. + properties: + path: + description: The JSON path that points to a field + that has drifted or has configuration differences. + type: string + valueInHub: + description: |- + The value at the JSON path from the hub cluster side. + + This field can be empty if the JSON path does not exist on the hub cluster side; i.e., + applying the manifest from the hub cluster side would remove the field. + type: string + valueInMember: + description: |- + The value at the JSON path from the member cluster side. + + This field can be empty if the JSON path does not exist on the member cluster side; i.e., + applying the manifest from the hub cluster side would add a new field. + type: string + required: + - path + type: object + type: array + observedInMemberClusterGeneration: + description: |- + ObservedInMemberClusterGeneration is the generation of the applied manifest on the member + cluster side. + + This might be nil if the resource has not been created yet in the member cluster. + format: int64 + type: integer + required: + - firstDiffedObservedTime + - observationTime + type: object + driftDetails: + description: |- + DriftDetails explains about the observed configuration drifts. + Fleet might truncate the details as appropriate to control object size. + + Note that configuration drifts can only occur on a resource if it is currently owned by + Fleet and its corresponding placement is set to use the ClientSideApply or ServerSideApply + apply strategy. In other words, DriftDetails and DiffDetails will not be populated + at the same time. + properties: + firstDriftedObservedTime: + description: FirstDriftedObservedTime is the timestamp when + the drift was first detected. + format: date-time + type: string + observationTime: + description: ObservationTime is the timestamp when the drift + was last detected. + format: date-time + type: string + observedDrifts: + description: |- + ObservedDrifts describes each drifted field found from the applied manifest. + Fleet might truncate the details as appropriate to control object size. + + Each entry specifies how the live state (the state on the member cluster side) compares + against the desired state (the state kept in the hub cluster manifest). + items: + description: |- + PatchDetail describes a patch that explains an observed configuration drift or + difference. + + A patch detail can be transcribed as a JSON patch operation, as specified in RFC 6902. + properties: + path: + description: The JSON path that points to a field + that has drifted or has configuration differences. + type: string + valueInHub: + description: |- + The value at the JSON path from the hub cluster side. + + This field can be empty if the JSON path does not exist on the hub cluster side; i.e., + applying the manifest from the hub cluster side would remove the field. + type: string + valueInMember: + description: |- + The value at the JSON path from the member cluster side. + + This field can be empty if the JSON path does not exist on the member cluster side; i.e., + applying the manifest from the hub cluster side would add a new field. + type: string + required: + - path + type: object + type: array + observedInMemberClusterGeneration: + description: |- + ObservedInMemberClusterGeneration is the generation of the applied manifest on the member + cluster side. + format: int64 + type: integer + required: + - firstDriftedObservedTime + - observationTime + - observedInMemberClusterGeneration + type: object identifier: description: resourceId represents a identity of a resource linking to manifests in spec.