feat(defrag): EtcdDefrag controller

README.md

@@ -76,7 +76,7 @@ For step-by-step setup, RBAC, image versions, and teardown see [docs/installatio
 - **[Installation](docs/installation.md)** — deploy the operator, create your first cluster, networking pitfalls, upgrades.
 - **[Concepts](docs/concepts.md)** — design rationale: locking pattern, single-seed bootstrap, GenerateName naming, scale-to-zero mechanics, conditions reference.
 - **[Operations](docs/operations.md)** — runbook for day-2: scaling, pausing/resuming, decoding conditions, escalating stuck reconciles, broken-member recovery.
-- **[Defragmentation](docs/etcd-defrag.md)** — the `EtcdDefrag` resource: reclaiming etcd backend disk and its safety model (API type; reconciling controller is a follow-up).
+- **[Defragmentation](docs/etcd-defrag.md)** — the `EtcdDefrag` resource: reclaiming etcd backend disk, one-shot and driven from outside on a schedule, and its safety model.
 - **[Migration](docs/migration.md)** — moving onto this operator from the legacy aenix operator; tracks behavioural changes that need an explicit migration step — currently the BYO root-credentials requirement when enabling auth.
 
 ## Testing

api/v1alpha2/etcddefrag_types.go

@@ -38,9 +38,8 @@ type EtcdDefragSpec struct {
 
 	// TTLSecondsAfterFinished records how long after a terminal phase this
 	// object should be garbage-collected — meaningful for objects a scheduler
-	// stamps out. NOTE: acted on by the (not-yet-implemented) reconciling
-	// controller; the API server does not garbage-collect custom resources on
-	// its own. Absent means the record is kept.
+	// stamps out. Acted on by the reconciling controller; the API server does not
+	// garbage-collect custom resources on its own. Absent means the record is kept.
 	// +kubebuilder:validation:Minimum=0
 	// +optional
 	TTLSecondsAfterFinished *int32 `json:"ttlSecondsAfterFinished,omitempty"`
@@ -71,9 +70,10 @@ type DefragRule struct {
 	// QuotaUsageAbove: when DbSize exceeds this fraction of the backend quota
 	// (approaching NOSPACE), lower the reclaimable floor to MinReclaim so small
 	// wins are taken under pressure. A member is never defragmented when its
-	// reclaimable space is below MinReclaim. Integer percent 1..100 with a "%"
-	// suffix, e.g. "80%".
-	// +kubebuilder:validation:Pattern=`^([1-9][0-9]?|100)%$`
+	// reclaimable space is below MinReclaim. Integer percent 1..99 with a "%"
+	// suffix, e.g. "80%"; 100% is rejected because a backend never exceeds its
+	// quota (etcd raises NOSPACE first), so the arm could never fire.
+	// +kubebuilder:validation:Pattern=`^[1-9][0-9]?%$`
 	// +optional
 	QuotaUsageAbove string `json:"quotaUsageAbove,omitempty"`
 
@@ -201,10 +201,6 @@ type MemberDefragStatus struct {
 // run-to-completion defragmentation of an EtcdCluster's members. Like
 // EtcdSnapshot it is a record: the operator drives it through status.phase and
 // it never re-runs.
-//
-// NOTE: this ships the API type ahead of its reconciling controller. Until that
-// controller lands, an EtcdDefrag is inert — creating one records intent but
-// nothing acts on it (no sweep runs, status stays empty, TTL does not fire).
 type EtcdDefrag struct {
 	metav1.TypeMeta   `json:",inline"`
 	metav1.ObjectMeta `json:"metadata,omitempty"`

charts/etcd-operator/crd-bases/etcd-operator.cozystack.io_etcddefrags.yaml

@@ -35,10 +35,6 @@ spec:
           run-to-completion defragmentation of an EtcdCluster's members. Like
           EtcdSnapshot it is a record: the operator drives it through status.phase and
           it never re-runs.
-
-          NOTE: this ships the API type ahead of its reconciling controller. Until that
-          controller lands, an EtcdDefrag is inert — creating one records intent but
-          nothing acts on it (no sweep runs, status stays empty, TTL does not fire).
         properties:
           apiVersion:
             description: |-
@@ -114,9 +110,10 @@ spec:
                       QuotaUsageAbove: when DbSize exceeds this fraction of the backend quota
                       (approaching NOSPACE), lower the reclaimable floor to MinReclaim so small
                       wins are taken under pressure. A member is never defragmented when its
-                      reclaimable space is below MinReclaim. Integer percent 1..100 with a "%"
-                      suffix, e.g. "80%".
-                    pattern: ^([1-9][0-9]?|100)%$
+                      reclaimable space is below MinReclaim. Integer percent 1..99 with a "%"
+                      suffix, e.g. "80%"; 100% is rejected because a backend never exceeds its
+                      quota (etcd raises NOSPACE first), so the arm could never fire.
+                    pattern: ^[1-9][0-9]?%$
                     type: string
                 type: object
                 x-kubernetes-validations:
@@ -136,9 +133,8 @@ spec:
                 description: |-
                   TTLSecondsAfterFinished records how long after a terminal phase this
                   object should be garbage-collected — meaningful for objects a scheduler
-                  stamps out. NOTE: acted on by the (not-yet-implemented) reconciling
-                  controller; the API server does not garbage-collect custom resources on
-                  its own. Absent means the record is kept.
+                  stamps out. Acted on by the reconciling controller; the API server does not
+                  garbage-collect custom resources on its own. Absent means the record is kept.
                 format: int32
                 minimum: 0
                 type: integer

charts/etcd-operator/files/manager-role-rules.yaml

@@ -1,3 +1,10 @@
+- apiGroups:
+    - ""
+  resources:
+    - events
+  verbs:
+    - create
+    - patch
 - apiGroups:
     - ""
   resources:
@@ -87,12 +94,24 @@
     - etcd-operator.cozystack.io
   resources:
     - etcdclusters/status
+    - etcddefrags/status
     - etcdmembers/status
     - etcdsnapshots/status
   verbs:
     - get
     - patch
     - update
+- apiGroups:
+    - etcd-operator.cozystack.io
+  resources:
+    - etcddefrags
+  verbs:
+    - delete
+    - get
+    - list
+    - patch
+    - update
+    - watch
 - apiGroups:
     - etcd-operator.cozystack.io
   resources:

controllers/etcd_client.go

@@ -35,13 +35,30 @@ type EtcdClusterClient interface {
 	MemberPromote(ctx context.Context, id uint64) (*clientv3.MemberPromoteResponse, error)
 	MemberRemove(ctx context.Context, id uint64) (*clientv3.MemberRemoveResponse, error)
 
-	// Status returns a single endpoint's server status, including the etcd
-	// version it is actually running (StatusResponse.Version). Used by the
-	// member controller to observe the running version into
-	// EtcdMember.status.version. *clientv3.Client satisfies this via its
+	// Status returns a single endpoint's server status: the etcd version it is
+	// actually running (StatusResponse.Version, observed into
+	// EtcdMember.status.version), its backend sizes (DbSize / DbSizeInUse), the
+	// leader it sees and any alarms — all read by the EtcdDefrag controller to
+	// gate and decide a defragmentation. *clientv3.Client satisfies this via its
 	// embedded Maintenance interface.
 	Status(ctx context.Context, endpoint string) (*clientv3.StatusResponse, error)
 
+	// Defragment releases a single endpoint's reclaimable backend space to the
+	// filesystem. It is per-endpoint (defrag is a member-local operation) and
+	// briefly blocks that member, so the EtcdDefrag controller calls it one
+	// member at a time on a healthy cluster. *clientv3.Client satisfies this via
+	// its embedded Maintenance interface.
+	Defragment(ctx context.Context, endpoint string) (*clientv3.DefragmentResponse, error)
+
+	// AlarmList and AlarmDisarm let the EtcdDefrag controller close the NOSPACE
+	// loop: a cluster at its backend quota raises NOSPACE and goes read-only,
+	// which is the case defrag exists to relieve, so the health gate permits the
+	// run — but the alarm stays armed after the space is reclaimed until it is
+	// explicitly disarmed. A CORRUPT alarm, by contrast, blocks the run.
+	// *clientv3.Client satisfies both via its embedded Maintenance interface.
+	AlarmList(ctx context.Context) (*clientv3.AlarmResponse, error)
+	AlarmDisarm(ctx context.Context, m *clientv3.AlarmMember) (*clientv3.AlarmResponse, error)
+
 	// Auth surface — used by reconcileAuth to provision the single root
 	// user/role and turn on authentication. The "root" role is built into
 	// etcd, so a RoleAdd is not needed: UserAdd("root", …) +