MCO-2599: makes bootc-operator aware of karpenter

api/v1alpha1/constants.go

@@ -30,4 +30,20 @@ const (
 	// state for which the controller considered emitting an event. It is
 	// observability bookkeeping only and is never used to drive reconciliation.
 	AnnotationLastObservedState = "bootc.dev/last-observed-state"
+
+	// AnnotationOwnsKarpenterDoNotRepair is set on a BootcNode to record that
+	// the controller added the karpenter.sh/do-not-repair annotation to the K8s
+	// Node during slot assignment. When present, the controller removes the
+	// karpenter.sh/do-not-repair annotation from the Node when the slot is freed.
+	// It is not set when the annotation was already present before slot assignment
+	// (externally set), so the controller never removes an annotation it did not add.
+	AnnotationOwnsKarpenterDoNotRepair = "bootc.dev/owns-karpenter-do-not-repair"
+
+	// AnnotationOwnsKarpenterDoNotDisrupt is set on a BootcNode to record that
+	// the controller added the karpenter.sh/do-not-disrupt annotation to the K8s
+	// Node during slot assignment. When present, the controller removes the
+	// karpenter.sh/do-not-disrupt annotation from the Node when the slot is freed.
+	// It is not set when the annotation was already present before slot assignment
+	// (externally set), so the controller never removes an annotation it did not add.
+	AnnotationOwnsKarpenterDoNotDisrupt = "bootc.dev/owns-karpenter-do-not-disrupt"
 )

docs/src/SUMMARY.md

@@ -8,3 +8,4 @@
 - [Helm Chart](helm.md)
 - [Operations](operations/index.md)
   - [Managing a pool](operations/pool.md)
+  - [Karpenter integration](operations/karpenter.md)

docs/src/operations/karpenter.md

@@ -0,0 +1,97 @@
+# Karpenter integration
+
+bootc-operator is aware of [Karpenter](https://karpenter.sh/) and coordinates
+with it during node updates to prevent unnecessary node replacement or
+voluntary disruption.
+
+## Why this matters
+
+Karpenter monitors nodes for health and may decide to replace or disrupt a
+node that appears unhealthy or underutilized — for example, one that is
+cordoned, being drained, or not yet Ready after a reboot. All of those
+conditions arise naturally during a bootc-operator update window, so without
+coordination Karpenter could race with the operator and terminate a node that
+is in the middle of a legitimate OS update.
+
+Karpenter respects two node-level annotations:
+
+- `karpenter.sh/do-not-repair: "true"` — prevents Node Auto Repair from
+  replacing a node that has unhealthy conditions.
+- `karpenter.sh/do-not-disrupt: "true"` — prevents voluntary disruption
+  actions (consolidation, drift) from selecting this node.
+
+> **Note:** `karpenter.sh/do-not-repair` was introduced in Karpenter v1.15.0.
+> That is the minimum required version for full bootc-operator Karpenter integration.
+
+bootc-operator sets both annotations automatically on Karpenter-managed nodes
+for the duration of the reboot slot.
+
+## How it works
+
+When the controller assigns a reboot slot to a node it checks whether the K8s
+Node object has an owner reference pointing to a Karpenter `NodeClaim`
+(`apiVersion: karpenter.sh/v1`, `kind: NodeClaim`). If it does, the
+controller takes the following steps:
+
+1. For each annotation (`do-not-repair`, `do-not-disrupt`): if the annotation
+   is **not** already present on the Node, the controller adds it (value
+   `"true"`) and records ownership by setting the corresponding tracking
+   annotation on the `BootcNode` object:
+   - `bootc.dev/owns-karpenter-do-not-repair: "true"`
+   - `bootc.dev/owns-karpenter-do-not-disrupt: "true"`
+2. The node proceeds through the normal update flow: drain, reboot, image
+   verification.
+3. When the slot is freed (node is Ready and running the new image), the
+   controller checks the tracking annotations on the `BootcNode`. For each
+   annotation it owns, it removes the corresponding annotation from the Node
+   and clears the tracking annotation from the `BootcNode`. Both removals are
+   issued as a single Node patch.
+
+## Pre-existing annotations
+
+If either Karpenter annotation is already on the Node before slot assignment —
+set by an external actor, another controller, or a manual operation —
+bootc-operator leaves it completely alone:
+
+- It does **not** overwrite or change the annotation on assignment.
+- It does **not** set the corresponding tracking annotation on the `BootcNode`.
+- It does **not** remove the annotation when the slot is freed.
+
+The two annotations are tracked independently. If only one is pre-existing, the
+operator owns (and later removes) only the other.
+
+This ensures bootc-operator never removes an annotation it did not add.
+
+## Limitations
+
+`karpenter.sh/do-not-disrupt` only blocks voluntary disruption methods
+(consolidation and drift). It does not protect a node against forceful
+termination triggered by expiration (`expireAfter`), interruption (spot
+instance reclaim), Node Auto Repair, or manual deletion. If any of those
+events occur while a node is in a bootc-operator reboot slot, Karpenter may
+still terminate it.
+
+`karpenter.sh/do-not-repair` similarly has no effect on voluntary disruption
+or expiration — it only suppresses Node Auto Repair.
+
+If a node is terminated by expiration, interruption, or any other forceful
+method while a bootc-operator update is in progress, the termination is caused
+by Karpenter acting on its own policy — not by the bootc-operator update. The
+update simply happened to be in flight at the time.
+
+Refer to the [Karpenter disruption documentation](https://karpenter.sh/docs/concepts/disruption/)
+for a full breakdown of which controls apply to which disruption methods.
+
+## Non-Karpenter nodes
+
+Nodes without a `NodeClaim` owner reference are unaffected. The feature
+activates only when the owner reference check matches.
+
+## Annotation reference
+
+| Annotation | Object | Managed by | Purpose |
+|---|---|---|---|
+| `karpenter.sh/do-not-repair` | K8s Node | bootc-operator (or external) | Prevents Karpenter Node Auto Repair from replacing this node during an update |
+| `karpenter.sh/do-not-disrupt` | K8s Node | bootc-operator (or external) | Prevents Karpenter voluntary disruption (consolidation, drift) during an update |
+| `bootc.dev/owns-karpenter-do-not-repair` | BootcNode | bootc-operator | Tracks that the operator added `do-not-repair`; cleared when the annotation is removed on slot free |
+| `bootc.dev/owns-karpenter-do-not-disrupt` | BootcNode | bootc-operator | Tracks that the operator added `do-not-disrupt`; cleared when the annotation is removed on slot free |

internal/controller/bootcnodepool_controller.go

@@ -859,8 +859,8 @@ func (r *BootcNodePoolReconciler) removeBootcNode(
 	if err := r.Get(ctx, types.NamespacedName{Name: bn.Name}, &node); err == nil {
 		// No point in cleaning up the Node object if it's going away anyway...
 		if node.DeletionTimestamp == nil {
-			if err := r.restoreCordonState(ctx, bn, &node); err != nil {
-				return fmt.Errorf("restoring cordon state: %w", err)
+			if err := r.restoreNodeState(ctx, bn, &node); err != nil {
+				return fmt.Errorf("restoring node state: %w", err)
 			}
 			if err := r.ensureManagedLabel(ctx, &node, false); err != nil {
 				return fmt.Errorf("removing managed label: %w", err)
@@ -877,21 +877,43 @@ func (r *BootcNodePoolReconciler) removeBootcNode(
 	return nil
 }
 
-// restoreCordonState uncordons the Node if the BootcNode's was-cordoned
-// annotation indicates the operator cordoned it. If the annotation is absent
-// or "true" (node was already cordoned before us), this is a no-op.
-func (r *BootcNodePoolReconciler) restoreCordonState(
+// restoreNodeState removes any Karpenter annotations the controller owns and
+// restores the node's cordon state, combining both into a single patch when
+// needed. Karpenter annotation ownership is tracked via BootcNode annotations
+// rather than inspecting the Node directly, so an external actor removing them
+// doesn't cause spurious patches. The cordon is only removed if the was-cordoned
+// annotation records that the operator was the one who cordoned it.
+func (r *BootcNodePoolReconciler) restoreNodeState(
 	ctx context.Context,
 	bn *bootcv1alpha1.BootcNode,
 	node *corev1.Node,
 ) error {
-	if bn.Annotations[bootcv1alpha1.AnnotationWasCordoned] != "false" {
+	log := logf.FromContext(ctx)
+	modified := node.DeepCopy()
+	changed := false
+
+	if metav1.HasAnnotation(bn.ObjectMeta, bootcv1alpha1.AnnotationOwnsKarpenterDoNotRepair) {
+		log.Info("Removing karpenter.sh/do-not-repair from node", "node", node.Name)
+		delete(modified.Annotations, karpenterDoNotRepairAnnotationKey)
+		changed = true
+	}
+	if metav1.HasAnnotation(bn.ObjectMeta, bootcv1alpha1.AnnotationOwnsKarpenterDoNotDisrupt) {
+		log.Info("Removing karpenter.sh/do-not-disrupt from node", "node", node.Name)
+		delete(modified.Annotations, karpenterDoNotDisruptAnnotationKey)
+		changed = true
+	}
+	if metav1.HasAnnotation(bn.ObjectMeta, bootcv1alpha1.AnnotationWasCordoned) &&
+		bn.Annotations[bootcv1alpha1.AnnotationWasCordoned] == "false" {
+		log.Info("Uncordoning node", "node", node.Name)
+		modified.Spec.Unschedulable = false
+		changed = true
+	}
+
+	if !changed {
 		return nil
 	}
-	modified := node.DeepCopy()
-	modified.Spec.Unschedulable = false
 	if err := r.Patch(ctx, modified, client.StrategicMergeFrom(node)); err != nil {
-		return fmt.Errorf("uncordoning node: %w", err)
+		return fmt.Errorf("restoring node state: %w", err)
 	}
 	*node = *modified
 	return nil

internal/controller/rollout.go

@@ -34,6 +34,18 @@ import (
 // https://github.com/bootc-dev/bootc-operator/issues/99.
 const unhealthySlotHaltThreshold = 2
 
+// Karpenter integration constants. The controller checks for a NodeClaim owner
+// reference and manages karpenter.sh/do-not-repair and
+// karpenter.sh/do-not-disrupt to prevent Karpenter from replacing or
+// voluntarily disrupting a node while it is being updated.
+// See docs/src/operations/karpenter.md.
+const (
+	karpenterDoNotRepairAnnotationKey  = "karpenter.sh/do-not-repair"
+	karpenterDoNotDisruptAnnotationKey = "karpenter.sh/do-not-disrupt"
+	karpenterNodeClaimAPIVersion       = "karpenter.sh/v1"
+	karpenterNodeClaimKind             = "NodeClaim"
+)
+
 // rolloutState holds the classified BootcNodes for a single reconcile
 // pass.
 type rolloutState struct {
@@ -188,47 +200,126 @@ func (r *BootcNodePoolReconciler) driveRollout(
 	return rs, nil
 }
 
-// assignRebootSlot marks a BootcNode as occupying a reboot slot and
-// cordons the corresponding K8s Node. It sets the in-reboot-slot
-// annotation on the BootcNode, records prior cordon state in the
-// was-cordoned annotation, and cordons the node. All operations are
-// idempotent.
+// nodeIsOwnedByKarpenterNodeClaim reports whether the K8s Node has an owner
+// reference pointing to a Karpenter NodeClaim (karpenter.sh/v1).
+func nodeIsOwnedByKarpenterNodeClaim(node *corev1.Node) bool {
+	for _, ref := range node.OwnerReferences {
+		if ref.APIVersion == karpenterNodeClaimAPIVersion && ref.Kind == karpenterNodeClaimKind {
+			return true
+		}
+	}
+	return false
+}
+
+// assignRebootSlot marks a BootcNode as occupying a reboot slot and cordons
+// the corresponding K8s Node. It sets the in-reboot-slot annotation on the
+// BootcNode and records the prior cordon state in the was-cordoned annotation.
+// For nodes owned by a Karpenter NodeClaim, it also sets the
+// karpenter.sh/do-not-repair and karpenter.sh/do-not-disrupt annotations on
+// the K8s Node to prevent Karpenter from replacing or disrupting the node
+// during the update. Ownership of those annotations is tracked on the BootcNode
+// so that restoreNodeState can remove only the ones the controller added.
+// All operations are idempotent.
 func (r *BootcNodePoolReconciler) assignRebootSlot(
 	ctx context.Context,
 	bn *bootcv1alpha1.BootcNode,
 	node *corev1.Node,
 ) error {
 	log := logf.FromContext(ctx)
 
-	// Set annotations on the BootcNode if not already present.
+	// Determine Karpenter intent once so both patch blocks stay consistent.
+	// shouldOwn* is true when the node is Karpenter-managed and the respective
+	// annotation is not already present (set by an external actor). In that
+	// case, the controller adds the annotation and records ownership so it can
+	// be cleaned up when the slot is freed.
+	karpenterOwned := nodeIsOwnedByKarpenterNodeClaim(node)
+	shouldOwnDoNotRepair := karpenterOwned &&
+		!metav1.HasAnnotation(node.ObjectMeta, karpenterDoNotRepairAnnotationKey)
+	shouldOwnDoNotDisrupt := karpenterOwned &&
+		!metav1.HasAnnotation(node.ObjectMeta, karpenterDoNotDisruptAnnotationKey)
+
+	// Patch the BootcNode with slot annotations if not already present.
 	if !metav1.HasAnnotation(bn.ObjectMeta, bootcv1alpha1.AnnotationInRebootSlot) {
 		log.Info("Assigning reboot slot", "node", bn.Name)
 		modifiedBN := bn.DeepCopy()
-		if modifiedBN.Annotations == nil {
-			modifiedBN.Annotations = map[string]string{}
-		}
-		modifiedBN.Annotations[bootcv1alpha1.AnnotationInRebootSlot] = ""
+
+		metav1.SetMetaDataAnnotation(
+			&modifiedBN.ObjectMeta,
+			bootcv1alpha1.AnnotationInRebootSlot,
+			"",
+		)
+
 		// Record whether the node was already cordoned before us.
 		if node.Spec.Unschedulable {
-			modifiedBN.Annotations[bootcv1alpha1.AnnotationWasCordoned] = "true"
+			metav1.SetMetaDataAnnotation(
+				&modifiedBN.ObjectMeta,
+				bootcv1alpha1.AnnotationWasCordoned,
+				"true",
+			)
 		} else {
-			modifiedBN.Annotations[bootcv1alpha1.AnnotationWasCordoned] = "false"
+			metav1.SetMetaDataAnnotation(
+				&modifiedBN.ObjectMeta,
+				bootcv1alpha1.AnnotationWasCordoned,
+				"false",
+			)
+		}
+		// Track Karpenter annotation ownership so freeRebootSlot knows
+		// whether to remove the annotations from the Node.
+		if shouldOwnDoNotRepair {
+			metav1.SetMetaDataAnnotation(
+				&modifiedBN.ObjectMeta,
+				bootcv1alpha1.AnnotationOwnsKarpenterDoNotRepair,
+				"true",
+			)
+		}
+		if shouldOwnDoNotDisrupt {
+			metav1.SetMetaDataAnnotation(
+				&modifiedBN.ObjectMeta,
+				bootcv1alpha1.AnnotationOwnsKarpenterDoNotDisrupt,
+				"true",
+			)
 		}
 		if err := r.Patch(ctx, modifiedBN, client.MergeFrom(bn)); err != nil {
 			return fmt.Errorf("annotating BootcNode: %w", err)
 		}
-		*bn = *modifiedBN
 	}
 
-	// Cordon the K8s Node if not already cordoned.
+	// If the node is cordoned and we do not own either of the Karpenter
+	// annotations, there is nothing for to do.
+	if node.Spec.Unschedulable && !shouldOwnDoNotRepair && !shouldOwnDoNotDisrupt {
+		return nil
+	}
+
+	// Cordon the K8s node and set the appropriate annotations. All operations
+	// are idempotent and are combined into a single API call.
+	modifiedNode := node.DeepCopy()
 	if !node.Spec.Unschedulable {
 		log.Info("Cordoning node", "node", node.Name)
-		modifiedNode := node.DeepCopy()
 		modifiedNode.Spec.Unschedulable = true
-		if err := r.Patch(ctx, modifiedNode, client.StrategicMergeFrom(node)); err != nil {
-			return fmt.Errorf("cordoning node: %w", err)
-		}
-		*node = *modifiedNode
+	}
+
+	// If we own the do-not-repair annotation, set it now.
+	if shouldOwnDoNotRepair {
+		log.Info("Setting karpenter.sh/do-not-repair on node", "node", node.Name)
+		metav1.SetMetaDataAnnotation(
+			&modifiedNode.ObjectMeta,
+			karpenterDoNotRepairAnnotationKey,
+			"true",
+		)
+	}
+
+	// If we own the do-not-disrupt annotation, set it now.
+	if shouldOwnDoNotDisrupt {
+		log.Info("Setting karpenter.sh/do-not-disrupt on node", "node", node.Name)
+		metav1.SetMetaDataAnnotation(
+			&modifiedNode.ObjectMeta,
+			karpenterDoNotDisruptAnnotationKey,
+			"true",
+		)
+	}
+
+	if err := r.Patch(ctx, modifiedNode, client.StrategicMergeFrom(node)); err != nil {
+		return fmt.Errorf("patching node: %w", err)
 	}
 
 	return nil
@@ -272,12 +363,15 @@ func (r *BootcNodePoolReconciler) freeRebootSlot(
 	bn *bootcv1alpha1.BootcNode,
 	node *corev1.Node,
 ) error {
-	if err := r.restoreCordonState(ctx, bn, node); err != nil {
+	if err := r.restoreNodeState(ctx, bn, node); err != nil {
 		return err
 	}
 
-	// Remove both reboot slot annotations from the BootcNode.
+	// Remove all reboot slot annotations from the BootcNode, including the
+	// Karpenter ownership trackers if present.
 	modified := bn.DeepCopy()
+	delete(modified.Annotations, bootcv1alpha1.AnnotationOwnsKarpenterDoNotRepair)
+	delete(modified.Annotations, bootcv1alpha1.AnnotationOwnsKarpenterDoNotDisrupt)
 	delete(modified.Annotations, bootcv1alpha1.AnnotationWasCordoned)
 	delete(modified.Annotations, bootcv1alpha1.AnnotationInRebootSlot)
 	if err := r.Patch(ctx, modified, client.MergeFrom(bn)); err != nil {