Emit Kubernetes events from the operator controllers

config/rbac/role.yaml

@@ -39,6 +39,13 @@ rules:
   - services/status
   verbs:
   - get
+- apiGroups:
+  - ""
+  resources:
+  - events
+  verbs:
+  - create
+  - patch
 - apiGroups:
   - ""
   resources:

controllers/event_recorder.go

@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package controllers
+
+import (
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/client-go/tools/record"
+)
+
+// noOpEventRecorder is a record.EventRecorder that discards all events.
+// It is used as a default so that a reconciler's Recorder is never nil,
+// allowing event call-sites to invoke it unconditionally.
+type noOpEventRecorder struct{}
+
+var _ record.EventRecorder = noOpEventRecorder{}
+
+func (noOpEventRecorder) Event(_ runtime.Object, _, _, _ string) {}
+
+func (noOpEventRecorder) Eventf(_ runtime.Object, _, _, _ string, _ ...interface{}) {}
+
+func (noOpEventRecorder) AnnotatedEventf(_ runtime.Object, _ map[string]string, _, _, _ string, _ ...interface{}) {
+}

controllers/events_test.go

@@ -0,0 +1,298 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package controllers
+
+import (
+	"context"
+	"errors"
+	"strings"
+	"testing"
+
+	solrv1beta1 "github.com/apache/solr-operator/api/v1beta1"
+	"github.com/apache/solr-operator/controllers/util"
+	"github.com/go-logr/logr"
+	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/resource"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
+	"k8s.io/client-go/tools/record"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/client/fake"
+	"sigs.k8s.io/controller-runtime/pkg/client/interceptor"
+)
+
+// errForcedPatchFailure is returned by the fake client to force the status patch to fail,
+// so that the "could not patch readiness condition" event path is exercised.
+var errForcedPatchFailure = errors.New("forced patch failure")
+
+// requireEvent asserts that one event was recorded whose type and reason match.
+// record.FakeRecorder formats each event as "<eventtype> <reason> <message>".
+func requireEvent(t *testing.T, rec *record.FakeRecorder, wantType, wantReason string) {
+	t.Helper()
+	select {
+	case got := <-rec.Events:
+		if !strings.HasPrefix(got, wantType+" "+wantReason+" ") {
+			t.Errorf("expected event of type %q with reason %q, got %q", wantType, wantReason, got)
+		}
+	default:
+		t.Fatalf("expected an event with reason %q to be recorded, but none was", wantReason)
+	}
+}
+
+// requireNoEvent asserts that no event is currently buffered on the recorder.
+func requireNoEvent(t *testing.T, rec *record.FakeRecorder) {
+	t.Helper()
+	select {
+	case got := <-rec.Events:
+		t.Fatalf("expected no event to be recorded, but got %q", got)
+	default:
+	}
+}
+
+// solrCloudWithStorageRequest builds a minimal SolrCloud requesting the given persistent data size.
+func solrCloudWithStorageRequest(size string) *solrv1beta1.SolrCloud {
+	return &solrv1beta1.SolrCloud{
+		ObjectMeta: metav1.ObjectMeta{Name: "test", Namespace: "test"},
+		Spec: solrv1beta1.SolrCloudSpec{
+			StorageOptions: solrv1beta1.SolrDataStorageOptions{
+				PersistentStorage: &solrv1beta1.SolrPersistentDataStorageOptions{
+					PersistentVolumeClaimTemplate: solrv1beta1.PersistentVolumeClaimTemplate{
+						Spec: corev1.PersistentVolumeClaimSpec{
+							Resources: corev1.VolumeResourceRequirements{
+								Requests: corev1.ResourceList{
+									corev1.ResourceStorage: resource.MustParse(size),
+								},
+							},
+						},
+					},
+				},
+			},
+		},
+	}
+}
+
+// podWithReadinessGate builds a Pod that advertises the given readiness gate (and no status yet).
+func podWithReadinessGate(condType corev1.PodConditionType) *corev1.Pod {
+	return &corev1.Pod{
+		ObjectMeta: metav1.ObjectMeta{Name: "test-pod", Namespace: "test"},
+		Spec: corev1.PodSpec{
+			ReadinessGates: []corev1.PodReadinessGate{{ConditionType: condType}},
+		},
+	}
+}
+
+// failingStatusPatchClient returns a fake client whose Status().Patch always fails, so that
+// readiness-condition patch failures (and their events) can be exercised deterministically.
+func failingStatusPatchClient() client.Client {
+	return fake.NewClientBuilder().
+		WithScheme(clientgoscheme.Scheme).
+		WithInterceptorFuncs(interceptor.Funcs{
+			SubResourcePatch: func(_ context.Context, _ client.Client, _ string, _ client.Object, _ client.Patch, _ ...client.SubResourcePatchOption) error {
+				return errForcedPatchFailure
+			},
+		}).
+		Build()
+}
+
+// TestDeterminePvcExpansionEmitsErrorEventOnBadAnnotation verifies a PVCExpansionError warning is
+// emitted when the existing minimum-size annotation recorded on the StatefulSet cannot be parsed.
+func TestDeterminePvcExpansionEmitsErrorEventOnBadAnnotation(t *testing.T) {
+	rec := record.NewFakeRecorder(8)
+	r := &SolrCloudReconciler{Recorder: rec}
+	instance := solrCloudWithStorageRequest("5Gi")
+	sts := &appsv1.StatefulSet{}
+	sts.Annotations = map[string]string{util.StorageMinimumSizeAnnotation: "not-a-quantity"}
+
+	clusterOp, _, err := determinePvcExpansionClusterOpLockIfNecessary(context.Background(), r, instance, sts, logr.Discard())
+	if err == nil {
+		t.Error("expected an error parsing the existing PVC size annotation, got nil")
+	}
+	if clusterOp != nil {
+		t.Errorf("expected no cluster operation to be started, got %+v", clusterOp)
+	}
+	requireEvent(t, rec, corev1.EventTypeWarning, "PVCExpansionError")
+}
+
+// TestDeterminePvcExpansionEmitsForbiddenEventOnShrink verifies a PVCExpansionForbidden warning is
+// emitted (and no cluster op started) when the requested size is smaller than the existing size.
+func TestDeterminePvcExpansionEmitsForbiddenEventOnShrink(t *testing.T) {
+	rec := record.NewFakeRecorder(8)
+	r := &SolrCloudReconciler{Recorder: rec}
+	instance := solrCloudWithStorageRequest("5Gi")
+	sts := &appsv1.StatefulSet{}
+	sts.Annotations = map[string]string{util.StorageMinimumSizeAnnotation: "10Gi"}
+
+	clusterOp, _, err := determinePvcExpansionClusterOpLockIfNecessary(context.Background(), r, instance, sts, logr.Discard())
+	if err != nil {
+		t.Errorf("did not expect an error for a shrink request, got %v", err)
+	}
+	if clusterOp != nil {
+		t.Errorf("expected no cluster operation for a shrink request, got %+v", clusterOp)
+	}
+	requireEvent(t, rec, corev1.EventTypeWarning, "PVCExpansionForbidden")
+}
+
+// TestDeterminePvcExpansionNoEventWhenSizeUnchanged verifies that the steady-state path (requested
+// size matches the recorded size) neither starts a cluster op nor records an event.
+func TestDeterminePvcExpansionNoEventWhenSizeUnchanged(t *testing.T) {
+	rec := record.NewFakeRecorder(8)
+	r := &SolrCloudReconciler{Recorder: rec}
+	instance := solrCloudWithStorageRequest("5Gi")
+	sts := &appsv1.StatefulSet{}
+	sts.Annotations = map[string]string{util.StorageMinimumSizeAnnotation: "5Gi"}
+
+	clusterOp, _, err := determinePvcExpansionClusterOpLockIfNecessary(context.Background(), r, instance, sts, logr.Discard())
+	if err != nil {
+		t.Errorf("did not expect an error, got %v", err)
+	}
+	if clusterOp != nil {
+		t.Errorf("expected no cluster operation, got %+v", clusterOp)
+	}
+	requireNoEvent(t, rec)
+}
+
+// TestHandleManagedScaleDownEmitsEventOnBadMetadata verifies a ClusterOperationError warning is
+// emitted when the scale-down target stored in the cluster-operation metadata cannot be parsed.
+func TestHandleManagedScaleDownEmitsEventOnBadMetadata(t *testing.T) {
+	rec := record.NewFakeRecorder(8)
+	r := &SolrCloudReconciler{Recorder: rec}
+	instance := &solrv1beta1.SolrCloud{ObjectMeta: metav1.ObjectMeta{Name: "test", Namespace: "test"}}
+	clusterOp := &SolrClusterOp{Operation: ScaleDownLock, Metadata: "not-an-int"}
+
+	_, _, _, err := handleManagedCloudScaleDown(context.Background(), r, instance, &appsv1.StatefulSet{}, clusterOp, nil, logr.Discard())
+	if err == nil {
+		t.Error("expected an error parsing the scale-down metadata, got nil")
+	}
+	requireEvent(t, rec, corev1.EventTypeWarning, "ClusterOperationError")
+}
+
+// TestInitializePodEmitsEventOnPatchFailure verifies that a failed readiness-condition patch while
+// starting traffic on a pod surfaces a PodReadinessConditionUpdateFailed warning.
+func TestInitializePodEmitsEventOnPatchFailure(t *testing.T) {
+	rec := record.NewFakeRecorder(8)
+	r := &SolrCloudReconciler{Client: failingStatusPatchClient(), Recorder: rec}
+	instance := &solrv1beta1.SolrCloud{ObjectMeta: metav1.ObjectMeta{Name: "test", Namespace: "test"}}
+	pod := podWithReadinessGate(util.SolrIsNotStoppedReadinessCondition)
+
+	if _, err := r.initializePod(context.Background(), instance, pod, logr.Discard()); err == nil {
+		t.Error("expected the forced patch failure to be returned, got nil")
+	}
+	requireEvent(t, rec, corev1.EventTypeWarning, "PodReadinessConditionUpdateFailed")
+}
+
+// TestEnsurePodReadinessConditionsEmitsEventOnPatchFailure verifies that a failed readiness-condition
+// patch while stopping traffic on a pod surfaces a PodReadinessConditionUpdateFailed warning.
+func TestEnsurePodReadinessConditionsEmitsEventOnPatchFailure(t *testing.T) {
+	rec := record.NewFakeRecorder(8)
+	r := &SolrCloudReconciler{Client: failingStatusPatchClient(), Recorder: rec}
+	instance := &solrv1beta1.SolrCloud{ObjectMeta: metav1.ObjectMeta{Name: "test", Namespace: "test"}}
+
+	pod := podWithReadinessGate(util.SolrIsNotStoppedReadinessCondition)
+	// Seed an existing condition with a different reason so a change (and thus a patch) is required.
+	pod.Status.Conditions = []corev1.PodCondition{{
+		Type:   util.SolrIsNotStoppedReadinessCondition,
+		Status: corev1.ConditionTrue,
+		Reason: string(PodStarted),
+	}}
+	ensureConditions := map[corev1.PodConditionType]podReadinessConditionChange{
+		util.SolrIsNotStoppedReadinessCondition: {
+			reason:  PodUpdate,
+			message: "Pod is being deleted, traffic to the pod must be stopped",
+			status:  false,
+		},
+	}
+
+	if _, err := EnsurePodReadinessConditions(context.Background(), r, instance, pod, ensureConditions, logr.Discard()); err == nil {
+		t.Error("expected the forced patch failure to be returned, got nil")
+	}
+	requireEvent(t, rec, corev1.EventTypeWarning, "PodReadinessConditionUpdateFailed")
+}
+
+// TestHandleManagedScaleUpEmitsEventOnBadMetadata verifies a ClusterOperationError warning is
+// emitted when the scale-up target stored in the cluster-operation metadata cannot be parsed.
+func TestHandleManagedScaleUpEmitsEventOnBadMetadata(t *testing.T) {
+	rec := record.NewFakeRecorder(8)
+	r := &SolrCloudReconciler{Recorder: rec}
+	instance := &solrv1beta1.SolrCloud{ObjectMeta: metav1.ObjectMeta{Name: "test", Namespace: "test"}}
+	clusterOp := &SolrClusterOp{Operation: ScaleUpLock, Metadata: "not-an-int"}
+
+	if _, _, err := handleManagedCloudScaleUp(context.Background(), r, instance, &appsv1.StatefulSet{}, clusterOp, nil, logr.Discard()); err == nil {
+		t.Error("expected an error parsing the scale-up metadata, got nil")
+	}
+	requireEvent(t, rec, corev1.EventTypeWarning, "ClusterOperationError")
+}
+
+// TestHandlePvcExpansionEmitsEventOnBadMetadata verifies a PVCExpansionError warning is emitted
+// when the target PVC size stored in the cluster-operation metadata cannot be parsed.
+func TestHandlePvcExpansionEmitsEventOnBadMetadata(t *testing.T) {
+	rec := record.NewFakeRecorder(8)
+	r := &SolrCloudReconciler{Recorder: rec}
+	instance := &solrv1beta1.SolrCloud{ObjectMeta: metav1.ObjectMeta{Name: "test", Namespace: "test"}}
+	clusterOp := &SolrClusterOp{Operation: PvcExpansionLock, Metadata: "not-a-quantity"}
+
+	if _, _, err := handlePvcExpansion(context.Background(), r, instance, &appsv1.StatefulSet{}, clusterOp, logr.Discard()); err == nil {
+		t.Error("expected an error parsing the PVC expansion metadata, got nil")
+	}
+	requireEvent(t, rec, corev1.EventTypeWarning, "PVCExpansionError")
+}
+
+// TestReconcileSolrCloudBackupEmitsCloudNotReadyEvent verifies a BackupCloudNotReady warning is
+// emitted when a backup is attempted against a repository that the SolrCloud has not yet marked
+// available. A GCS repository is used so that EnsureDirectoryForBackup is a no-op (no pod exec).
+func TestReconcileSolrCloudBackupEmitsCloudNotReadyEvent(t *testing.T) {
+	scheme := runtime.NewScheme()
+	if err := clientgoscheme.AddToScheme(scheme); err != nil {
+		t.Fatalf("could not add client-go types to scheme: %v", err)
+	}
+	if err := solrv1beta1.AddToScheme(scheme); err != nil {
+		t.Fatalf("could not add solr types to scheme: %v", err)
+	}
+
+	solrCloud := &solrv1beta1.SolrCloud{
+		ObjectMeta: metav1.ObjectMeta{Name: "cloud", Namespace: "test"},
+		Spec: solrv1beta1.SolrCloudSpec{
+			BackupRepositories: []solrv1beta1.SolrBackupRepository{{
+				Name: "test-repo",
+				GCS:  &solrv1beta1.GcsRepository{Bucket: "test-bucket"},
+			}},
+		},
+		Status: solrv1beta1.SolrCloudStatus{
+			BackupRepositoriesAvailable: map[string]bool{"test-repo": false},
+		},
+	}
+	backup := &solrv1beta1.SolrBackup{
+		ObjectMeta: metav1.ObjectMeta{Name: "backup", Namespace: "test"},
+		Spec: solrv1beta1.SolrBackupSpec{
+			SolrCloud:      "cloud",
+			RepositoryName: "test-repo",
+		},
+	}
+
+	rec := record.NewFakeRecorder(8)
+	r := &SolrBackupReconciler{
+		Client:   fake.NewClientBuilder().WithScheme(scheme).WithObjects(solrCloud).Build(),
+		Recorder: rec,
+	}
+
+	if _, _, err := r.reconcileSolrCloudBackup(context.Background(), backup, &backup.Status.IndividualSolrBackupStatus, logr.Discard()); err == nil {
+		t.Error("expected a 'cloud not ready' error, got nil")
+	}
+	requireEvent(t, rec, corev1.EventTypeWarning, "BackupCloudNotReady")
+}