diff --git a/pkg/monitortestlibrary/pathologicaleventlibrary/duplicated_event_patterns.go b/pkg/monitortestlibrary/pathologicaleventlibrary/duplicated_event_patterns.go index 3a5fc6c18b3f..78ab3de9c88a 100644 --- a/pkg/monitortestlibrary/pathologicaleventlibrary/duplicated_event_patterns.go +++ b/pkg/monitortestlibrary/pathologicaleventlibrary/duplicated_event_patterns.go @@ -544,8 +544,10 @@ func NewUniversalPathologicalEventMatchers(kubeConfig *rest.Config, finalInterva singleNodeConnectionRefusedMatcher := newSingleNodeConnectionRefusedEventMatcher(finalIntervals) singleNodeKubeAPIServerProgressingMatcher := newSingleNodeKubeAPIProgressingEventMatcher(finalIntervals) + singleNodeOperatorAPIConnectionRefusedMatcher := newSingleNodeOperatorAPIConnectionRefusedDuringKubeAPIProgressingEventMatcher(finalIntervals) registry.AddPathologicalEventMatcherOrDie(singleNodeConnectionRefusedMatcher) registry.AddPathologicalEventMatcherOrDie(singleNodeKubeAPIServerProgressingMatcher) + registry.AddPathologicalEventMatcherOrDie(singleNodeOperatorAPIConnectionRefusedMatcher) vsphereConfigurationTestsRollOutTooOftenMatcher := newVsphereConfigurationTestsRollOutTooOftenEventMatcher(finalIntervals) registry.AddPathologicalEventMatcherOrDie(vsphereConfigurationTestsRollOutTooOftenMatcher) @@ -913,16 +915,7 @@ func IsDuringAPIServerProgressingOnSNO(topology string, events monitorapi.Interv if topology != "single" { return func(eventInterval monitorapi.Interval) bool { return false } } - ocpKubeAPIServerProgressingInterval := events.Filter(func(interval monitorapi.Interval) bool { - isNodeInstaller := interval.Message.Reason == monitorapi.NodeInstallerReason - isOperatorSource := interval.Source == monitorapi.SourceOperatorState - isKubeAPI := interval.Locator.Keys[monitorapi.LocatorClusterOperatorKey] == "kube-apiserver" - - isKubeAPIInstaller := isNodeInstaller && isOperatorSource && isKubeAPI - isKubeAPIInstallProgressing := isKubeAPIInstaller && interval.Message.Annotations[monitorapi.AnnotationCondition] == "Progressing" - - return isKubeAPIInstallProgressing - }) + ocpKubeAPIServerProgressingInterval := events.Filter(isKubeAPIServerNodeInstallerProgressing) return func(i monitorapi.Interval) bool { for _, progressingInterval := range ocpKubeAPIServerProgressingInterval { @@ -1202,17 +1195,7 @@ func newSingleNodeConnectionRefusedEventMatcher(finalIntervals monitorapi.Interv func newSingleNodeKubeAPIProgressingEventMatcher(finalIntervals monitorapi.Intervals) EventMatcher { snoTopology := v1.SingleReplicaTopologyMode - ocpKubeAPIServerProgressingInterval := finalIntervals.Filter(func(eventInterval monitorapi.Interval) bool { - - isNodeInstaller := eventInterval.Message.Reason == monitorapi.NodeInstallerReason - isOperatorSource := eventInterval.Source == monitorapi.SourceOperatorState - isKubeAPI := eventInterval.Locator.Keys[monitorapi.LocatorClusterOperatorKey] == "kube-apiserver" - - isKubeAPIInstaller := isNodeInstaller && isOperatorSource && isKubeAPI - isKubeAPIInstallProgressing := isKubeAPIInstaller && eventInterval.Message.Annotations[monitorapi.AnnotationCondition] == "Progressing" - - return isKubeAPIInstallProgressing - }) + ocpKubeAPIServerProgressingInterval := finalIntervals.Filter(isKubeAPIServerNodeInstallerProgressing) // We buffer 1 second since Before and After are not inclusive for time comparisons. for i := range ocpKubeAPIServerProgressingInterval { @@ -1233,6 +1216,33 @@ func newSingleNodeKubeAPIProgressingEventMatcher(finalIntervals monitorapi.Inter } } +// Operators updating status against the kubernetes API spam connection-refused events while kube-apiserver is rolled on SNO. +// Those kube events often carry a firstTimestamp from an earlier roll (e.g. install) and accumulate counts across later +// rolls, so interval containment cannot allow them. If any KAS Progressing interval is observed, allow the matched +// events for the job (same approach as TopologyAwareHintsDisabledDuringTaintManagerTests). +func newSingleNodeOperatorAPIConnectionRefusedDuringKubeAPIProgressingEventMatcher(finalIntervals monitorapi.Intervals) EventMatcher { + snoTopology := v1.SingleReplicaTopologyMode + + matcher := &SimplePathologicalEventMatcher{ + name: "OperatorAPIConnectionRefusedDuringSingleNodeKubeAPIProgressing", + messageReasonRegex: regexp.MustCompile(`^(EtcdEndpointsErrorUpdatingStatus|ScriptControllerErrorUpdatingStatus|OperatorDegraded: MachineConfigNodeFailed)$`), + messageHumanRegex: regexp.MustCompile(`dial tcp .* connect: connection refused`), + topology: &snoTopology, + } + if len(finalIntervals.Filter(isKubeAPIServerNodeInstallerProgressing)) == 0 { + matcher.neverAllow = true + } + return matcher +} + +func isKubeAPIServerNodeInstallerProgressing(eventInterval monitorapi.Interval) bool { + isNodeInstaller := eventInterval.Message.Reason == monitorapi.NodeInstallerReason + isOperatorSource := eventInterval.Source == monitorapi.SourceOperatorState + isKubeAPI := eventInterval.Locator.Keys[monitorapi.LocatorClusterOperatorKey] == "kube-apiserver" + return isNodeInstaller && isOperatorSource && isKubeAPI && + eventInterval.Message.Annotations[monitorapi.AnnotationCondition] == "Progressing" +} + func newDeferringOperatorNodeUpdateTooOftenEventMatcher(finalIntervals monitorapi.Intervals) EventMatcher { DeferringOperatorNodeUpdateIntervals := finalIntervals.Filter(func(eventInterval monitorapi.Interval) bool { return eventInterval.Source == monitorapi.SourceE2ETest && diff --git a/pkg/monitortestlibrary/pathologicaleventlibrary/duplicated_events_test.go b/pkg/monitortestlibrary/pathologicaleventlibrary/duplicated_events_test.go index 75ad4115159f..3efc587e7a43 100644 --- a/pkg/monitortestlibrary/pathologicaleventlibrary/duplicated_events_test.go +++ b/pkg/monitortestlibrary/pathologicaleventlibrary/duplicated_events_test.go @@ -454,6 +454,120 @@ func TestPathologicalEventsWithNamespaces(t *testing.T) { topology: v1.HighlyAvailableTopologyMode, expectedMessage: "1 events happened too frequently\n\nevent happened 22 times, something is wrong: - ns/mynamespace reason/FailedScheduling 0/6 nodes are available: 2 node(s) were unschedulable, 4 node(s) didn't match pod anti-affinity rules. preemption: 0/6 nodes are available: 2 Preemption is not helpful for scheduling, 4 No preemption victims found for incoming pod.. (04:00:00Z) result=reject ", }, + { + name: "ignore etcd-operator API connection refused on SNO when kube-apiserver is Progressing", + intervals: []monitorapi.Interval{ + { + Condition: monitorapi.Condition{ + Level: monitorapi.Info, + Locator: monitorapi.Locator{ + Keys: map[monitorapi.LocatorKey]string{ + monitorapi.LocatorClusterOperatorKey: "kube-apiserver", + }, + }, + Message: monitorapi.Message{ + Reason: monitorapi.NodeInstallerReason, + HumanMessage: "NodeInstallerProgressing: 1 node is at revision 7; 0 nodes have achieved new revision 8", + Annotations: map[monitorapi.AnnotationKey]string{ + monitorapi.AnnotationCondition: "Progressing", + monitorapi.AnnotationStatus: "True", + }, + }, + }, + Source: monitorapi.SourceOperatorState, + From: from.Add(-1 * time.Minute), + To: to.Add(1 * time.Minute), + }, + monitorapi.NewInterval(monitorapi.SourceKubeEvent, monitorapi.Info). + Locator(monitorapi.Locator{Keys: map[monitorapi.LocatorKey]string{ + monitorapi.LocatorNamespaceKey: "openshift-etcd-operator", + }}).Message( + monitorapi.NewMessage().Reason("EtcdEndpointsErrorUpdatingStatus"). + HumanMessage(`Put "https://172.30.0.1:443/apis/operator.openshift.io/v1/etcds/cluster/status": dial tcp 172.30.0.1:443: connect: connection refused`). + WithAnnotation(monitorapi.AnnotationCount, "42")). + Build(from, to), + }, + namespace: "openshift-etcd-operator", + platform: v1.AWSPlatformType, + topology: v1.SingleReplicaTopologyMode, + expectedMessage: "", + }, + { + name: "ignore MCO API connection refused on SNO when kube-apiserver is Progressing", + intervals: []monitorapi.Interval{ + { + Condition: monitorapi.Condition{ + Level: monitorapi.Info, + Locator: monitorapi.Locator{ + Keys: map[monitorapi.LocatorKey]string{ + monitorapi.LocatorClusterOperatorKey: "kube-apiserver", + }, + }, + Message: monitorapi.Message{ + Reason: monitorapi.NodeInstallerReason, + HumanMessage: "NodeInstallerProgressing: 1 node is at revision 9; 0 nodes have achieved new revision 10", + Annotations: map[monitorapi.AnnotationKey]string{ + monitorapi.AnnotationCondition: "Progressing", + monitorapi.AnnotationStatus: "True", + }, + }, + }, + Source: monitorapi.SourceOperatorState, + From: from.Add(-1 * time.Minute), + To: to.Add(1 * time.Minute), + }, + monitorapi.NewInterval(monitorapi.SourceKubeEvent, monitorapi.Info). + Locator(monitorapi.Locator{Keys: map[monitorapi.LocatorKey]string{ + monitorapi.LocatorNamespaceKey: "openshift-machine-config-operator", + }}).Message( + monitorapi.NewMessage().Reason("OperatorDegraded: MachineConfigNodeFailed"). + HumanMessage(`Failed to resync 5.0.0-0.nightly-2026-07-30-185227 because: Get "https://172.30.0.1:443/apis/machineconfiguration.openshift.io/v1/machineconfignodes": dial tcp 172.30.0.1:443: connect: connection refused`). + WithAnnotation(monitorapi.AnnotationCount, "37")). + Build(from, to), + }, + namespace: "openshift-machine-config-operator", + platform: v1.AWSPlatformType, + topology: v1.SingleReplicaTopologyMode, + expectedMessage: "", + }, + { + name: "match etcd-operator API connection refused on HA even when kube-apiserver is Progressing", + intervals: []monitorapi.Interval{ + { + Condition: monitorapi.Condition{ + Level: monitorapi.Info, + Locator: monitorapi.Locator{ + Keys: map[monitorapi.LocatorKey]string{ + monitorapi.LocatorClusterOperatorKey: "kube-apiserver", + }, + }, + Message: monitorapi.Message{ + Reason: monitorapi.NodeInstallerReason, + HumanMessage: "NodeInstallerProgressing: 1 node is at revision 7; 0 nodes have achieved new revision 8", + Annotations: map[monitorapi.AnnotationKey]string{ + monitorapi.AnnotationCondition: "Progressing", + monitorapi.AnnotationStatus: "True", + }, + }, + }, + Source: monitorapi.SourceOperatorState, + From: from.Add(-1 * time.Minute), + To: to.Add(1 * time.Minute), + }, + monitorapi.NewInterval(monitorapi.SourceKubeEvent, monitorapi.Info). + Locator(monitorapi.Locator{Keys: map[monitorapi.LocatorKey]string{ + monitorapi.LocatorNamespaceKey: "openshift-etcd-operator", + }}).Message( + monitorapi.NewMessage().Reason("EtcdEndpointsErrorUpdatingStatus"). + HumanMessage(`Put "https://172.30.0.1:443/apis/operator.openshift.io/v1/etcds/cluster/status": dial tcp 172.30.0.1:443: connect: connection refused`). + WithAnnotation(monitorapi.AnnotationCount, "42")). + Build(from, to), + }, + namespace: "openshift-etcd-operator", + platform: v1.AWSPlatformType, + topology: v1.HighlyAvailableTopologyMode, + expectedMessage: "1 events happened too frequently\n\nevent happened 42 times, something is wrong: namespace/openshift-etcd-operator - reason/EtcdEndpointsErrorUpdatingStatus Put \"https://172.30.0.1:443/apis/operator.openshift.io/v1/etcds/cluster/status\": dial tcp 172.30.0.1:443: connect: connection refused (04:00:00Z) result=reject ", + }, } for _, test := range tests {