Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -231,6 +231,8 @@ OK - 1 Alerts: 0 Firing - 0 Pending - 1 Inactive
\_[OK] [ApacheDown] is inactive
```
Values support regular expressions (e.g. `severity=warn.+`).
#### Checking watchdog alerts
In Prometheus a "watchdog" or "dead man's switch" is an alert that is always firing to ensure alerting pipeline is working. The `-W, --watchdog` flag can be used to flip/negate the exit state of the plugin for these kind of alerts:
Expand Down
40 changes: 29 additions & 11 deletions cmd/alert.go
Original file line number Diff line number Diff line change
Expand Up @@ -120,7 +120,11 @@ inactive = 0`,
}
}

labelsMatchedInclude := matchesLabel(rl.AlertingRule.Labels, cliAlertConfig.IncludeLabels)
labelsMatchedInclude, regexErr := matchesLabel(rl.AlertingRule.Labels, cliAlertConfig.IncludeLabels)

if regexErr != nil {
check.ExitRaw(check.Unknown, "Invalid regular expression provided:", regexErr.Error())
}

if len(cliAlertConfig.IncludeLabels) > 0 && !labelsMatchedInclude {
// If the alert labels don't match here we can skip it.
Expand All @@ -143,7 +147,11 @@ inactive = 0`,
continue
}

labelsMatchedExclude := matchesLabel(rl.AlertingRule.Labels, cliAlertConfig.ExcludeLabels)
labelsMatchedExclude, regexErr := matchesLabel(rl.AlertingRule.Labels, cliAlertConfig.ExcludeLabels)

if regexErr != nil {
check.ExitRaw(check.Unknown, "Invalid regular expression provided:", regexErr.Error())
}

if len(cliAlertConfig.ExcludeLabels) > 0 && labelsMatchedExclude {
// If the alert labels matches here we can skip it.
Expand Down Expand Up @@ -261,12 +269,12 @@ func init() {

fs.StringArrayVar(&cliAlertConfig.IncludeLabels, "include-label", []string{},
"The label of one or more specific alerts to include. "+
"\nThis parameter can be repeated e.g.: '--include-label prio=high --include-label another=example'"+
"\nThis parameter can be repeated e.g.: '--include-label prio=high --include-label another=example'. Supports regex for values"+
"\nNote that repeated --include-label are combined using a union.")

fs.StringArrayVar(&cliAlertConfig.ExcludeLabels, "exclude-label", []string{},
"The label of one or more specific alerts to exclude."+
"\nThis parameter can be repeated e.g.: '--exclude-label prio=high --exclude-label another=example'")
"\nThis parameter can be repeated e.g.: '--exclude-label prio=high --exclude-label another=example'. Supports regex for values")

fs.BoolVarP(&cliAlertConfig.ProblemsOnly, "problems", "P", false,
"Display only alerts which status is not inactive/OK. Note that in combination with the --name flag this might result in no alerts being displayed")
Expand Down Expand Up @@ -314,22 +322,32 @@ func matches(input string, regexToExclude []string) (bool, error) {
}

// Matches a list of labels against a list of labels
func matchesLabel(labels model.LabelSet, labelsToMatch []string) bool {
func matchesLabel(labels model.LabelSet, labelsToMatch []string) (bool, error) {
for _, lb := range labelsToMatch {
kv := strings.SplitN(lb, "=", 2)
expectedLabelSet := strings.SplitN(lb, "=", 2)

if len(kv) != 2 {
if len(expectedLabelSet) != 2 {
continue
}
// Do we have a value for the expected key?
actualValue, ok := labels[model.LabelName(expectedLabelSet[0])]

key, value := model.LabelName(kv[0]), model.LabelValue(kv[1])
if !ok {
return false, nil
}

if val, ok := labels[key]; ok && val == value {
return true
re, err := regexp.Compile(expectedLabelSet[1])
if err != nil {
return false, err
}

// Does the values match the expected label regex?
if re.MatchString(string(actualValue)) {
return true, nil
}
}

return false
return false, nil
}

// negateStatus turns an OK state into critical and a warning/critical state into OK
Expand Down
30 changes: 30 additions & 0 deletions cmd/alert_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -269,6 +269,20 @@ exit status 2
\_ [WARNING] [SqlAccessDeniedRate] - Job: [mysql] on Instance: [localhost] is pending - value: 0.40 - {"alertname":"SqlAccessDeniedRate","instance":"localhost","job":"mysql","severity":"warning"}
|total=1 firing=0 pending=1 inactive=0

exit status 1
`,
},
{
name: "alert-exclude-label-regex",
server: httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusOK)
w.Write(loadTestdata(alertTestDataSet1))
})),
args: []string{"run", "../main.go", "alert", "--exclude-label", "severity=crit.*"},
expected: `[WARNING] - 1 Alerts: 0 Firing - 1 Pending - 0 Inactive
\_ [WARNING] [SqlAccessDeniedRate] - Job: [mysql] on Instance: [localhost] is pending - value: 0.40 - {"alertname":"SqlAccessDeniedRate","instance":"localhost","job":"mysql","severity":"warning"}
|total=1 firing=0 pending=1 inactive=0

exit status 1
`,
},
Expand All @@ -285,6 +299,22 @@ exit status 1
\_ [CRITICAL] [BlackboxTLS] - Job: [blackbox] on Instance: [https://localhost:443] is firing - value: -6065338.00 - {"alertname":"TLS","instance":"https://localhost:443","job":"blackbox","severity":"critical"}
|total=3 firing=1 pending=1 inactive=1

exit status 2
`,
},
{
name: "alert-include-label-multiple-regex",
server: httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusOK)
w.Write(loadTestdata(alertTestDataSet1))
})),
args: []string{"run", "../main.go", "alert", "--include-label", "team=data.+", "--include-label", "severity=critical"},
expected: `[CRITICAL] - 3 Alerts: 1 Firing - 1 Pending - 1 Inactive
\_ [OK] [HostOutOfMemory] is inactive
\_ [WARNING] [SqlAccessDeniedRate] - Job: [mysql] on Instance: [localhost] is pending - value: 0.40 - {"alertname":"SqlAccessDeniedRate","instance":"localhost","job":"mysql","severity":"warning"}
\_ [CRITICAL] [BlackboxTLS] - Job: [blackbox] on Instance: [https://localhost:443] is firing - value: -6065338.00 - {"alertname":"TLS","instance":"https://localhost:443","job":"blackbox","severity":"critical"}
|total=3 firing=1 pending=1 inactive=1

exit status 2
`,
},
Expand Down