Skip to content

WIP: Allow operator API connection-refused events on SNO during KAS Progressing #31482

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
eggfoobar wants to merge 1 commit into
base: main
Choose a base branch
from
+145 −21
Open

WIP: Allow operator API connection-refused events on SNO during KAS Progressing #31482

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
52 changes: 31 additions & 21 deletions pkg/monitortestlibrary/pathologicaleventlibrary/duplicated_event_patterns.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -544,8 +544,10 @@ func NewUniversalPathologicalEventMatchers(kubeConfig *rest.Config, finalInterva

singleNodeConnectionRefusedMatcher := newSingleNodeConnectionRefusedEventMatcher(finalIntervals)
singleNodeKubeAPIServerProgressingMatcher := newSingleNodeKubeAPIProgressingEventMatcher(finalIntervals)
singleNodeOperatorAPIConnectionRefusedMatcher := newSingleNodeOperatorAPIConnectionRefusedDuringKubeAPIProgressingEventMatcher(finalIntervals)
registry.AddPathologicalEventMatcherOrDie(singleNodeConnectionRefusedMatcher)
registry.AddPathologicalEventMatcherOrDie(singleNodeKubeAPIServerProgressingMatcher)
registry.AddPathologicalEventMatcherOrDie(singleNodeOperatorAPIConnectionRefusedMatcher)

vsphereConfigurationTestsRollOutTooOftenMatcher := newVsphereConfigurationTestsRollOutTooOftenEventMatcher(finalIntervals)
registry.AddPathologicalEventMatcherOrDie(vsphereConfigurationTestsRollOutTooOftenMatcher)
Expand Down Expand Up @@ -913,16 +915,7 @@ func IsDuringAPIServerProgressingOnSNO(topology string, events monitorapi.Interv
if topology != "single" {
return func(eventInterval monitorapi.Interval) bool { return false }
}
ocpKubeAPIServerProgressingInterval := events.Filter(func(interval monitorapi.Interval) bool {
isNodeInstaller := interval.Message.Reason == monitorapi.NodeInstallerReason
isOperatorSource := interval.Source == monitorapi.SourceOperatorState
isKubeAPI := interval.Locator.Keys[monitorapi.LocatorClusterOperatorKey] == "kube-apiserver"

isKubeAPIInstaller := isNodeInstaller && isOperatorSource && isKubeAPI
isKubeAPIInstallProgressing := isKubeAPIInstaller && interval.Message.Annotations[monitorapi.AnnotationCondition] == "Progressing"

return isKubeAPIInstallProgressing
})
ocpKubeAPIServerProgressingInterval := events.Filter(isKubeAPIServerNodeInstallerProgressing)

return func(i monitorapi.Interval) bool {
for _, progressingInterval := range ocpKubeAPIServerProgressingInterval {
Expand Down Expand Up @@ -1202,17 +1195,7 @@ func newSingleNodeConnectionRefusedEventMatcher(finalIntervals monitorapi.Interv
func newSingleNodeKubeAPIProgressingEventMatcher(finalIntervals monitorapi.Intervals) EventMatcher {
snoTopology := v1.SingleReplicaTopologyMode

ocpKubeAPIServerProgressingInterval := finalIntervals.Filter(func(eventInterval monitorapi.Interval) bool {

isNodeInstaller := eventInterval.Message.Reason == monitorapi.NodeInstallerReason
isOperatorSource := eventInterval.Source == monitorapi.SourceOperatorState
isKubeAPI := eventInterval.Locator.Keys[monitorapi.LocatorClusterOperatorKey] == "kube-apiserver"

isKubeAPIInstaller := isNodeInstaller && isOperatorSource && isKubeAPI
isKubeAPIInstallProgressing := isKubeAPIInstaller && eventInterval.Message.Annotations[monitorapi.AnnotationCondition] == "Progressing"

return isKubeAPIInstallProgressing
})
ocpKubeAPIServerProgressingInterval := finalIntervals.Filter(isKubeAPIServerNodeInstallerProgressing)

// We buffer 1 second since Before and After are not inclusive for time comparisons.
for i := range ocpKubeAPIServerProgressingInterval {
Expand All @@ -1233,6 +1216,33 @@ func newSingleNodeKubeAPIProgressingEventMatcher(finalIntervals monitorapi.Inter
}
}

// Operators updating status against the kubernetes API spam connection-refused events while kube-apiserver is rolled on SNO.
// Those kube events often carry a firstTimestamp from an earlier roll (e.g. install) and accumulate counts across later
// rolls, so interval containment cannot allow them. If any KAS Progressing interval is observed, allow the matched
// events for the job (same approach as TopologyAwareHintsDisabledDuringTaintManagerTests).
func newSingleNodeOperatorAPIConnectionRefusedDuringKubeAPIProgressingEventMatcher(finalIntervals monitorapi.Intervals) EventMatcher {
snoTopology := v1.SingleReplicaTopologyMode

matcher := &SimplePathologicalEventMatcher{
name: "OperatorAPIConnectionRefusedDuringSingleNodeKubeAPIProgressing",
messageReasonRegex: regexp.MustCompile(`^(EtcdEndpointsErrorUpdatingStatus|ScriptControllerErrorUpdatingStatus|OperatorDegraded: MachineConfigNodeFailed)$`),
messageHumanRegex: regexp.MustCompile(`dial tcp .* connect: connection refused`),
topology: &snoTopology,
}
if len(finalIntervals.Filter(isKubeAPIServerNodeInstallerProgressing)) == 0 {
matcher.neverAllow = true
}
return matcher
}

func isKubeAPIServerNodeInstallerProgressing(eventInterval monitorapi.Interval) bool {
isNodeInstaller := eventInterval.Message.Reason == monitorapi.NodeInstallerReason
isOperatorSource := eventInterval.Source == monitorapi.SourceOperatorState
isKubeAPI := eventInterval.Locator.Keys[monitorapi.LocatorClusterOperatorKey] == "kube-apiserver"
return isNodeInstaller && isOperatorSource && isKubeAPI &&
eventInterval.Message.Annotations[monitorapi.AnnotationCondition] == "Progressing"
}

func newDeferringOperatorNodeUpdateTooOftenEventMatcher(finalIntervals monitorapi.Intervals) EventMatcher {
DeferringOperatorNodeUpdateIntervals := finalIntervals.Filter(func(eventInterval monitorapi.Interval) bool {
return eventInterval.Source == monitorapi.SourceE2ETest &&
Expand Down
114 changes: 114 additions & 0 deletions pkg/monitortestlibrary/pathologicaleventlibrary/duplicated_events_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -454,6 +454,120 @@ func TestPathologicalEventsWithNamespaces(t *testing.T) {
topology: v1.HighlyAvailableTopologyMode,
expectedMessage: "1 events happened too frequently\n\nevent happened 22 times, something is wrong: - ns/mynamespace reason/FailedScheduling 0/6 nodes are available: 2 node(s) were unschedulable, 4 node(s) didn't match pod anti-affinity rules. preemption: 0/6 nodes are available: 2 Preemption is not helpful for scheduling, 4 No preemption victims found for incoming pod.. (04:00:00Z) result=reject ",
},
{
name: "ignore etcd-operator API connection refused on SNO when kube-apiserver is Progressing",
intervals: []monitorapi.Interval{
{
Condition: monitorapi.Condition{
Level: monitorapi.Info,
Locator: monitorapi.Locator{
Keys: map[monitorapi.LocatorKey]string{
monitorapi.LocatorClusterOperatorKey: "kube-apiserver",
},
},
Message: monitorapi.Message{
Reason: monitorapi.NodeInstallerReason,
HumanMessage: "NodeInstallerProgressing: 1 node is at revision 7; 0 nodes have achieved new revision 8",
Annotations: map[monitorapi.AnnotationKey]string{
monitorapi.AnnotationCondition: "Progressing",
monitorapi.AnnotationStatus: "True",
},
},
},
Source: monitorapi.SourceOperatorState,
From: from.Add(-1 * time.Minute),
To: to.Add(1 * time.Minute),
},
monitorapi.NewInterval(monitorapi.SourceKubeEvent, monitorapi.Info).
Locator(monitorapi.Locator{Keys: map[monitorapi.LocatorKey]string{
monitorapi.LocatorNamespaceKey: "openshift-etcd-operator",
}}).Message(
monitorapi.NewMessage().Reason("EtcdEndpointsErrorUpdatingStatus").
HumanMessage(`Put "https://172.30.0.1:443/apis/operator.openshift.io/v1/etcds/cluster/status": dial tcp 172.30.0.1:443: connect: connection refused`).
WithAnnotation(monitorapi.AnnotationCount, "42")).
Build(from, to),
},
namespace: "openshift-etcd-operator",
platform: v1.AWSPlatformType,
topology: v1.SingleReplicaTopologyMode,
expectedMessage: "",
},
{
name: "ignore MCO API connection refused on SNO when kube-apiserver is Progressing",
intervals: []monitorapi.Interval{
{
Condition: monitorapi.Condition{
Level: monitorapi.Info,
Locator: monitorapi.Locator{
Keys: map[monitorapi.LocatorKey]string{
monitorapi.LocatorClusterOperatorKey: "kube-apiserver",
},
},
Message: monitorapi.Message{
Reason: monitorapi.NodeInstallerReason,
HumanMessage: "NodeInstallerProgressing: 1 node is at revision 9; 0 nodes have achieved new revision 10",
Annotations: map[monitorapi.AnnotationKey]string{
monitorapi.AnnotationCondition: "Progressing",
monitorapi.AnnotationStatus: "True",
},
},
},
Source: monitorapi.SourceOperatorState,
From: from.Add(-1 * time.Minute),
To: to.Add(1 * time.Minute),
},
monitorapi.NewInterval(monitorapi.SourceKubeEvent, monitorapi.Info).
Locator(monitorapi.Locator{Keys: map[monitorapi.LocatorKey]string{
monitorapi.LocatorNamespaceKey: "openshift-machine-config-operator",
}}).Message(
monitorapi.NewMessage().Reason("OperatorDegraded: MachineConfigNodeFailed").
HumanMessage(`Failed to resync 5.0.0-0.nightly-2026-07-30-185227 because: Get "https://172.30.0.1:443/apis/machineconfiguration.openshift.io/v1/machineconfignodes": dial tcp 172.30.0.1:443: connect: connection refused`).
WithAnnotation(monitorapi.AnnotationCount, "37")).
Build(from, to),
},
namespace: "openshift-machine-config-operator",
platform: v1.AWSPlatformType,
topology: v1.SingleReplicaTopologyMode,
expectedMessage: "",
},
{
name: "match etcd-operator API connection refused on HA even when kube-apiserver is Progressing",
intervals: []monitorapi.Interval{
{
Condition: monitorapi.Condition{
Level: monitorapi.Info,
Locator: monitorapi.Locator{
Keys: map[monitorapi.LocatorKey]string{
monitorapi.LocatorClusterOperatorKey: "kube-apiserver",
},
},
Message: monitorapi.Message{
Reason: monitorapi.NodeInstallerReason,
HumanMessage: "NodeInstallerProgressing: 1 node is at revision 7; 0 nodes have achieved new revision 8",
Annotations: map[monitorapi.AnnotationKey]string{
monitorapi.AnnotationCondition: "Progressing",
monitorapi.AnnotationStatus: "True",
},
},
},
Source: monitorapi.SourceOperatorState,
From: from.Add(-1 * time.Minute),
To: to.Add(1 * time.Minute),
},
monitorapi.NewInterval(monitorapi.SourceKubeEvent, monitorapi.Info).
Locator(monitorapi.Locator{Keys: map[monitorapi.LocatorKey]string{
monitorapi.LocatorNamespaceKey: "openshift-etcd-operator",
}}).Message(
monitorapi.NewMessage().Reason("EtcdEndpointsErrorUpdatingStatus").
HumanMessage(`Put "https://172.30.0.1:443/apis/operator.openshift.io/v1/etcds/cluster/status": dial tcp 172.30.0.1:443: connect: connection refused`).
WithAnnotation(monitorapi.AnnotationCount, "42")).
Build(from, to),
},
namespace: "openshift-etcd-operator",
platform: v1.AWSPlatformType,
topology: v1.HighlyAvailableTopologyMode,
expectedMessage: "1 events happened too frequently\n\nevent happened 42 times, something is wrong: namespace/openshift-etcd-operator - reason/EtcdEndpointsErrorUpdatingStatus Put \"https://172.30.0.1:443/apis/operator.openshift.io/v1/etcds/cluster/status\": dial tcp 172.30.0.1:443: connect: connection refused (04:00:00Z) result=reject ",
},
}

for _, test := range tests {
Expand Down