Skip to content

Add job wait time metrics #152

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
jacek12345678 wants to merge 1 commit into
base: main
Choose a base branch
from
+72 −0
Open

Add job wait time metrics #152

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 23 additions & 0 deletions README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -93,6 +93,7 @@ Metrics
| `azure_devops_agentpool_size` | live | Number of agents per agent pool |
| `azure_devops_agentpool_usage` | live | Usage of agent pool (used agents; percent 0-1) |
| `azure_devops_agentpool_queue_length` | live | Queue length per agent pool |
| `azure_devops_agentpool_job_wait_seconds` | live | Agent pool job wait time in seconds before assignment to an agent |
| `azure_devops_agentpool_agent_info` | live | Agent information per agent pool |
| `azure_devops_agentpool_agent_status` | live | Status informations (eg. created date) for each agent in a agent pool |
| `azure_devops_agentpool_agent_job` | live | Currently running jobs on each agent |
Expand Down Expand Up @@ -184,3 +185,25 @@ azure_devops_agentpool_info
count by(agentPoolID) (azure_devops_agentpool_agent_info{status="online",enabled="true"})
)
```

Top 20 currently waiting jobs (minutes)
```
topk(
20,
azure_devops_agentpool_job_wait_seconds{state="queued"} / 60
)
```

Max queued wait time by pool (minutes)
```
max by (agentPoolID) (
azure_devops_agentpool_job_wait_seconds{state="queued"}
) / 60
```

Average queued wait time by pool (minutes)
```
avg by (agentPoolID) (
azure_devops_agentpool_job_wait_seconds{state="queued"}
) / 60
```
49 changes: 49 additions & 0 deletions metrics_agentpool.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package main
import (
"context"
"log/slog"
"time"

"github.com/prometheus/client_golang/prometheus"
"github.com/webdevops/go-common/prometheus/collector"
Expand All @@ -22,6 +23,7 @@ type MetricsCollectorAgentPool struct {
agentPoolAgentStatus *prometheus.GaugeVec
agentPoolAgentJob *prometheus.GaugeVec
agentPoolQueueLength *prometheus.GaugeVec
agentPoolJobWait *prometheus.GaugeVec
}
}

Expand Down Expand Up @@ -123,6 +125,23 @@ func (m *MetricsCollectorAgentPool) Setup(collector *collector.Collector) {
},
)
m.Collector.RegisterMetricList("agentPoolQueueLength", m.prometheus.agentPoolQueueLength, true)

m.prometheus.agentPoolJobWait = prometheus.NewGaugeVec(
prometheus.GaugeOpts{
Name: "azure_devops_agentpool_job_wait_seconds",
Help: "Azure DevOps agentpool job wait time before assignment to an agent",
},
[]string{
"agentPoolID",
"jobRequestId",
"definitionID",
"definitionName",
"planType",
"scopeID",
"state",
},
)
m.Collector.RegisterMetricList("agentPoolJobWait", m.prometheus.agentPoolJobWait, true)
}

func (m *MetricsCollectorAgentPool) Reset() {}
Expand Down Expand Up @@ -242,13 +261,43 @@ func (m *MetricsCollectorAgentPool) collectAgentPoolJobs(ctx context.Context, lo
}

agentPoolQueueLengthMetric := m.Collector.GetMetricList("agentPoolQueueLength")
agentPoolJobWaitMetric := m.Collector.GetMetricList("agentPoolJobWait")

notStartedJobCount := 0

for _, agentPoolJob := range list.List {
if agentPoolJob.AssignTime == nil {
notStartedJobCount++
}

if agentPoolJob.QueueTime.IsZero() {
continue
}

var waitSeconds float64
var state string

if agentPoolJob.AssignTime == nil {
waitSeconds = time.Since(agentPoolJob.QueueTime).Seconds()
state = "queued"
} else {
waitSeconds = agentPoolJob.AssignTime.Sub(agentPoolJob.QueueTime).Seconds()
state = "assigned"
}

if waitSeconds < 0 {
waitSeconds = 0
}

agentPoolJobWaitMetric.Add(prometheus.Labels{
"agentPoolID": int64ToString(agentPoolId),
"jobRequestId": int64ToString(agentPoolJob.RequestId),
"definitionID": int64ToString(agentPoolJob.Definition.Id),
"definitionName": agentPoolJob.Definition.Name,
"planType": agentPoolJob.PlanType,
"scopeID": agentPoolJob.ScopeId,
"state": state,
}, waitSeconds)
}

infoLabels := prometheus.Labels{
Expand Down