Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 11 additions & 2 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -103,6 +103,11 @@ rootly alerts ack <id>
rootly alerts resolve <id>
rootly alerts escalate <id>
rootly alerts escalate <id> --escalation-policy=<uuid> --level=2
rootly alerts escalate <id> --team=platform # Page a team on the alert

# Page a team for an incident (links the page to the incident)
rootly incidents escalate INC-123 --team=platform
rootly incidents escalate INC-123 --team=platform --escalation-policy=<uuid> --note="Database is down"

# Services
rootly services list
Expand Down Expand Up @@ -190,10 +195,14 @@ List a status page's templates with `rootly status-pages templates list --status

`status-pages events create` and `status-pages events update` accept incident statuses `investigating`, `identified`, `monitoring`, and `resolved`, plus scheduled-maintenance statuses `scheduled`, `in_progress`, and `completed`. For `status-pages events resolve`, `--status` defaults to `resolved`; use `--status=completed` to complete scheduled maintenance.

### Alert Escalation
### Escalation

`rootly alerts escalate <alert-id>` escalates an alert using its current policy. Optionally pass `--escalation-policy=<uuid>` and/or `--level=<n>` to select a policy or escalation level; levels must be at least 1. The API's error title is shown when escalation cannot proceed.

Pass `--team=<slug-or-id>` instead to page a team on the existing alert. `--team` cannot be combined with `--level`; `--escalation-policy` adds that policy as an extra paging target.

`rootly incidents escalate <incident-id> --team=<slug-or-id>` pages a team for an incident. The incident can be a UUID, slug, or sequential ID such as `INC-123`. It creates an alert that pages the team and links it to the incident. Optional `--escalation-policy=<uuid>` pages that policy as well, and `--note=<text>` is sent to the responders as the alert description. If the team was paged but linking the alert failed, the error prints the alert ID; retry only the link with `--link-alert=<alert-id>` so the team is not paged twice. Alerts already linked to the incident are kept. An unknown team or incident gives a clear error, and the API's error title is shown otherwise. Paging requires Rootly alerting to be enabled for the organization.

### Pulse Flags

| Flag | Short | Env Var | Description |
Expand Down Expand Up @@ -255,7 +264,7 @@ rootly incidents list --sort=-created_at

| Command | Subcommands | Aliases |
|---------|-------------|---------|
| `rootly incidents` | `list`, `get`, `create`, `update`, `delete` | `incident`, `inc` |
| `rootly incidents` | `list`, `get`, `create`, `update`, `delete`, `escalate` | `incident`, `inc` |
| `rootly alerts` | `list`, `get`, `create`, `update`, `ack`, `resolve`, `escalate` | `alert`, `alr` |
| `rootly form-fields` | `list` | |
| `rootly services` | `list`, `get`, `create`, `update`, `delete` | `service`, `svc` |
Expand Down
209 changes: 205 additions & 4 deletions internal/api/client.go
Original file line number Diff line number Diff line change
Expand Up @@ -2644,7 +2644,25 @@ func (c *Client) EscalateAlertCLI(ctx context.Context, id string, escalationPoli
if err != nil {
return fmt.Errorf("failed to read response: %w", err)
}
if httpResp.StatusCode >= http.StatusBadRequest && httpResp.StatusCode < http.StatusInternalServerError {
return apiStatusError(httpResp.StatusCode, body)
}

// Notification target types accepted by POST /v1/alerts.
const (
NotificationTargetGroup = "Group"
NotificationTargetEscalationPolicy = "EscalationPolicy"
)

// NotificationTarget is a team or escalation policy to page.
type NotificationTarget struct {
Type string `json:"type"`
ID string `json:"id"`
}

// apiStatusError returns nil for 2xx responses. For 4xx it surfaces the first
// JSON:API error title when present, otherwise just the status code.
func apiStatusError(status int, body []byte) error {
if status >= http.StatusBadRequest && status < http.StatusInternalServerError {
var response struct {
Errors []struct {
Title string `json:"title"`
Expand All @@ -2653,14 +2671,197 @@ func (c *Client) EscalateAlertCLI(ctx context.Context, id string, escalationPoli
if json.Unmarshal(body, &response) == nil && len(response.Errors) > 0 && response.Errors[0].Title != "" {
return fmt.Errorf("%s", response.Errors[0].Title)
}
return fmt.Errorf("API returned status %d", httpResp.StatusCode)
return fmt.Errorf("API returned status %d", status)
}
if httpResp.StatusCode < http.StatusOK || httpResp.StatusCode >= http.StatusMultipleChoices {
return fmt.Errorf("API returned status %d", httpResp.StatusCode)
if status < http.StatusOK || status >= http.StatusMultipleChoices {
return fmt.Errorf("API returned status %d", status)
}
return nil
}

// PageAlertTargetsCLI pages the given targets through POST /v1/alerts. An empty
// alertID creates a new alert; otherwise the existing alert is upserted. It
// returns the ID of the alert that paged the targets.
func (c *Client) PageAlertTargetsCLI(ctx context.Context, alertID, summary, description string, targets []NotificationTarget) (string, error) {
attributes := map[string]interface{}{
"notification_targets": targets,
}
// An existing alert keeps its summary: sending a stale one would overwrite
// concurrent edits. The server needs it only to create the alert.
if alertID == "" {
attributes["summary"] = summary
}
if description != "" {
attributes["description"] = description
}
data := map[string]interface{}{
"type": "alerts",
"attributes": attributes,
}
if alertID != "" {
// The server reads the upsert id from data.attributes only.
attributes["id"] = alertID
}
bodyBytes, err := json.Marshal(map[string]interface{}{"data": data})
if err != nil {
return "", fmt.Errorf("failed to marshal request body: %w", err)
}
req, err := http.NewRequestWithContext(ctx, http.MethodPost, c.endpoint+"/v1/alerts", strings.NewReader(string(bodyBytes)))
if err != nil {
return "", fmt.Errorf("failed to create request: %w", err)
}
req.Header.Set("Authorization", "Bearer "+c.apiKey)
req.Header.Set("Content-Type", "application/vnd.api+json")

httpResp, err := c.httpClient.Do(req)
if err != nil {
return "", fmt.Errorf("failed to page team: %w", err)
}
defer func() { _ = httpResp.Body.Close() }()
body, err := io.ReadAll(httpResp.Body)
if err != nil {
return "", fmt.Errorf("failed to read response: %w", err)
}
if err := apiStatusError(httpResp.StatusCode, body); err != nil {
return "", err
}
var response struct {
Data struct {
ID string `json:"id"`
} `json:"data"`
}
if err := json.Unmarshal(body, &response); err != nil {
return "", fmt.Errorf("failed to parse response: %w", err)
}
if response.Data.ID == "" {
return "", fmt.Errorf("failed to parse response: missing alert id")
}
return response.Data.ID, nil
}

// listIncidentAlertIDsCLI returns the IDs of every alert already linked to an incident.
func (c *Client) listIncidentAlertIDsCLI(ctx context.Context, incidentID string) ([]string, error) {
const pageSize = 100
var ids []string
for page := 1; ; page++ {
path := fmt.Sprintf("/v1/incidents/%s/alerts?page[number]=%d&page[size]=%d", neturl.PathEscape(incidentID), page, pageSize)
req, err := http.NewRequestWithContext(ctx, http.MethodGet, c.endpoint+path, http.NoBody)
if err != nil {
return nil, fmt.Errorf("failed to create request: %w", err)
}
req.Header.Set("Authorization", "Bearer "+c.apiKey)
req.Header.Set("Content-Type", "application/vnd.api+json")
httpResp, err := c.httpClient.Do(req)
if err != nil {
return nil, fmt.Errorf("failed to list incident alerts: %w", err)
}
body, err := io.ReadAll(httpResp.Body)
_ = httpResp.Body.Close()
if err != nil {
return nil, fmt.Errorf("failed to read response: %w", err)
}
if err := apiStatusError(httpResp.StatusCode, body); err != nil {
return nil, err
}
var response struct {
Data []struct {
ID string `json:"id"`
} `json:"data"`
}
if err := json.Unmarshal(body, &response); err != nil {
return nil, fmt.Errorf("failed to parse response: %w", err)
}
for _, a := range response.Data {
ids = append(ids, a.ID)
}
if len(response.Data) < pageSize {
return ids, nil
}
}
}

// attachAlertsMaxAttempts bounds how often the link is rewritten when a
// concurrent writer drops it.
const attachAlertsMaxAttempts = 2

// AttachAlertsToIncidentCLI links alerts to an incident. The endpoint replaces
// the incident's alert set, so the already linked alerts are sent along. A
// concurrent writer can overwrite the set between our read and write, so the
// links are read back afterwards and the write is repeated once if any are missing.
func (c *Client) AttachAlertsToIncidentCLI(ctx context.Context, incidentID string, alertIDs []string) error {
for attempt := 1; ; attempt++ {
if err := c.replaceIncidentAlertsCLI(ctx, incidentID, alertIDs); err != nil {
return err
}
linked, err := c.listIncidentAlertIDsCLI(ctx, incidentID)
if err != nil {
return err
}
if containsAll(linked, alertIDs) {
return nil
}
if attempt == attachAlertsMaxAttempts {
return fmt.Errorf("alerts %s are not linked to the incident after %d attempts; another update may have replaced them", strings.Join(alertIDs, ", "), attempt)
}
}
}

func containsAll(have, want []string) bool {
set := make(map[string]bool, len(have))
for _, id := range have {
set[id] = true
}
for _, id := range want {
if !set[id] {
return false
}
}
return true
}

// replaceIncidentAlertsCLI reads the incident's linked alerts and writes them back with alertIDs added.
func (c *Client) replaceIncidentAlertsCLI(ctx context.Context, incidentID string, alertIDs []string) error {
existing, err := c.listIncidentAlertIDsCLI(ctx, incidentID)
if err != nil {
return err
}
seen := make(map[string]bool, len(existing))
merged := make([]string, 0, len(existing)+len(alertIDs))
for _, id := range append(existing, alertIDs...) {
if !seen[id] {
seen[id] = true
merged = append(merged, id)
}
}
bodyBytes, err := json.Marshal(map[string]interface{}{
"data": map[string]interface{}{
"type": "alerts",
"attributes": map[string]interface{}{"alert_ids": merged},
},
Comment thread
greptile-apps[bot] marked this conversation as resolved.
Resolved
})
if err != nil {
return fmt.Errorf("failed to marshal request body: %w", err)
}
path := fmt.Sprintf("/v1/incidents/%s/alerts", neturl.PathEscape(incidentID))
req, err := http.NewRequestWithContext(ctx, http.MethodPost, c.endpoint+path, strings.NewReader(string(bodyBytes)))
if err != nil {
return fmt.Errorf("failed to create request: %w", err)
}
req.Header.Set("Authorization", "Bearer "+c.apiKey)
req.Header.Set("Content-Type", "application/vnd.api+json")

httpResp, err := c.httpClient.Do(req)
if err != nil {
return fmt.Errorf("failed to attach alerts to incident: %w", err)
}
defer func() { _ = httpResp.Body.Close() }()
body, err := io.ReadAll(httpResp.Body)
if err != nil {
return fmt.Errorf("failed to read response: %w", err)
}
return apiStatusError(httpResp.StatusCode, body)
}

// ResolveAlertCLI resolves an alert using raw HTTP POST.
func (c *Client) ResolveAlertCLI(ctx context.Context, id, resolutionMessage string, resolveIncidents bool) error {
// Build URL
Expand Down
Loading
Loading