From 817cac8158dc21194eb1c3b7c002983168541830 Mon Sep 17 00:00:00 2001
From: jaces <jacek12345678@users.noreply.github.com>
Date: Tue, 15 Sep 2026 14:48:14 +0200
Subject: [PATCH] Added job wait time metrics

---
 README.md            | 23 +++++++++++++++++++++
 metrics_agentpool.go | 49 ++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 72 insertions(+)

diff --git a/README.md b/README.md
index 32d7818..ce2a4e6 100644
--- a/README.md
+++ b/README.md
@@ -93,6 +93,7 @@ Metrics
 | `azure_devops_agentpool_size`                  | live          | Number of agents per agent pool                                                         |
 | `azure_devops_agentpool_usage`                 | live          | Usage of agent pool (used agents; percent 0-1)                                          |
 | `azure_devops_agentpool_queue_length`          | live          | Queue length per agent pool                                                             |
+| `azure_devops_agentpool_job_wait_seconds`      | live          | Agent pool job wait time in seconds before assignment to an agent                       |
 | `azure_devops_agentpool_agent_info`            | live          | Agent information per agent pool                                                        |
 | `azure_devops_agentpool_agent_status`          | live          | Status informations (eg. created date) for each agent in a agent pool                   |
 | `azure_devops_agentpool_agent_job`             | live          | Currently running jobs on each agent                                                    |
@@ -184,3 +185,25 @@ azure_devops_agentpool_info
   count by(agentPoolID) (azure_devops_agentpool_agent_info{status="online",enabled="true"})
 )
 ```
+
+Top 20 currently waiting jobs (minutes)
+```
+topk(
+  20,
+  azure_devops_agentpool_job_wait_seconds{state="queued"} / 60
+)
+```
+
+Max queued wait time by pool (minutes)
+```
+max by (agentPoolID) (
+  azure_devops_agentpool_job_wait_seconds{state="queued"}
+) / 60
+```
+
+Average queued wait time by pool (minutes)
+```
+avg by (agentPoolID) (
+  azure_devops_agentpool_job_wait_seconds{state="queued"}
+) / 60
+```
diff --git a/metrics_agentpool.go b/metrics_agentpool.go
index 6c26f58..32a7518 100644
--- a/metrics_agentpool.go
+++ b/metrics_agentpool.go
@@ -3,6 +3,7 @@ package main
 import (
 	"context"
 	"log/slog"
+	"time"
 
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/webdevops/go-common/prometheus/collector"
@@ -22,6 +23,7 @@ type MetricsCollectorAgentPool struct {
 		agentPoolAgentStatus *prometheus.GaugeVec
 		agentPoolAgentJob    *prometheus.GaugeVec
 		agentPoolQueueLength *prometheus.GaugeVec
+		agentPoolJobWait     *prometheus.GaugeVec
 	}
 }
 
@@ -123,6 +125,23 @@ func (m *MetricsCollectorAgentPool) Setup(collector *collector.Collector) {
 		},
 	)
 	m.Collector.RegisterMetricList("agentPoolQueueLength", m.prometheus.agentPoolQueueLength, true)
+
+	m.prometheus.agentPoolJobWait = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: "azure_devops_agentpool_job_wait_seconds",
+			Help: "Azure DevOps agentpool job wait time before assignment to an agent",
+		},
+		[]string{
+			"agentPoolID",
+			"jobRequestId",
+			"definitionID",
+			"definitionName",
+			"planType",
+			"scopeID",
+			"state",
+		},
+	)
+	m.Collector.RegisterMetricList("agentPoolJobWait", m.prometheus.agentPoolJobWait, true)
 }
 
 func (m *MetricsCollectorAgentPool) Reset() {}
@@ -242,6 +261,7 @@ func (m *MetricsCollectorAgentPool) collectAgentPoolJobs(ctx context.Context, lo
 	}
 
 	agentPoolQueueLengthMetric := m.Collector.GetMetricList("agentPoolQueueLength")
+	agentPoolJobWaitMetric := m.Collector.GetMetricList("agentPoolJobWait")
 
 	notStartedJobCount := 0
 
@@ -249,6 +269,35 @@ func (m *MetricsCollectorAgentPool) collectAgentPoolJobs(ctx context.Context, lo
 		if agentPoolJob.AssignTime == nil {
 			notStartedJobCount++
 		}
+
+		if agentPoolJob.QueueTime.IsZero() {
+			continue
+		}
+
+		var waitSeconds float64
+		var state string
+
+		if agentPoolJob.AssignTime == nil {
+			waitSeconds = time.Since(agentPoolJob.QueueTime).Seconds()
+			state = "queued"
+		} else {
+			waitSeconds = agentPoolJob.AssignTime.Sub(agentPoolJob.QueueTime).Seconds()
+			state = "assigned"
+		}
+
+		if waitSeconds < 0 {
+			waitSeconds = 0
+		}
+
+		agentPoolJobWaitMetric.Add(prometheus.Labels{
+			"agentPoolID":    int64ToString(agentPoolId),
+			"jobRequestId":   int64ToString(agentPoolJob.RequestId),
+			"definitionID":   int64ToString(agentPoolJob.Definition.Id),
+			"definitionName": agentPoolJob.Definition.Name,
+			"planType":       agentPoolJob.PlanType,
+			"scopeID":        agentPoolJob.ScopeId,
+			"state":          state,
+		}, waitSeconds)
 	}
 
 	infoLabels := prometheus.Labels{