Add restarts metric to windows services (#21840)

* Add state metric to windows services

* Changelog

* Validate metadata

* Add restarts metric to windows services

* Add unit tests

* Requested changes

* Update readme

* Fix changelog

* Fix tests

Author: briantu

windows_service/README.md

@@ -75,6 +75,8 @@ services:
     trigger_start: true
 ```
 
+Beginning with Agent version 7.74, the check automatically collects metrics for Windows services.
+
 #### Tags
 
 The check automatically tags the Windows service name to each service check in the `windows_service:<SERVICE>` tag. The `<SERVICE>` name in the tag uses lowercase and special characters are replaced with underscores. See [Getting Started with Tags][12] for more information.

windows_service/changelog.d/21840.added

@@ -0,0 +1 @@
+Add restarts metric to windows services

windows_service/datadog_checks/windows_service/windows_service.py

@@ -246,6 +246,21 @@ class WindowsService(AgentCheck):
     }
     UNKNOWN_LITERAL = "unknown"
 
+    def __init__(self, name, init_config, instances):
+        super().__init__(name, init_config, instances)
+        self._service_pid_cache: dict[str, int] = {}
+
+    def _get_service_restarts(self, service_name: str, current_pid: int) -> int:
+        if current_pid == 0:
+            return 0
+        prev_pid = self._service_pid_cache.get(service_name, None)
+        restarts = 0
+        if prev_pid is not None and prev_pid != current_pid:
+            restarts = 1
+        # only store the last running pid for the service
+        self._service_pid_cache[service_name] = current_pid
+        return restarts
+
     def check(self, instance):
         services = instance.get('services', [])
         custom_tags = instance.get('tags', [])
@@ -318,6 +333,8 @@ def check(self, instance):
             if service_pid != 0:
                 service_uptime = _get_process_uptime_from_cache(service_pid, process_cache)
 
+            service_restarts = self._get_service_restarts(service_name, service_pid)
+
             status = self.STATE_TO_STATUS.get(state, self.UNKNOWN)
             state_string = self.STATE_TO_STRING.get(state, self.UNKNOWN_LITERAL)
 
@@ -346,6 +363,7 @@ def check(self, instance):
             # Send 1 for windows_service.state so the user can sum by the windows_service_state tag
             # to filter services by state. e.g. sum:windows_service.state{*} by windows_service_state
             self.gauge('windows_service.state', 1, tags=tags)
+            self.count('windows_service.restarts', service_restarts, tags=tags)
 
         if 'ALL' not in services:
             for service in services_unseen:
@@ -370,3 +388,4 @@ def check(self, instance):
                 self.log.debug('service state for %s %s', service, status)
                 self.gauge('windows_service.uptime', 0, tags=tags)
                 self.gauge('windows_service.state', 1, tags=tags)
+                self.count('windows_service.restarts', 0, tags=tags)

windows_service/metadata.csv

@@ -1,3 +1,4 @@
 metric_name,metric_type,interval,unit_name,per_unit_name,description,orientation,integration,short_name,curated_metric,sample_tags
+windows_service.restarts,count,,event,,The number of restarts of the host process of the Windows service,0,windows_service,restarts,,
 windows_service.state,gauge,,service,,Sum by state to count the number of services in each state,0,windows_service,state,,
 windows_service.uptime,gauge,,second,,The uptime (in seconds) of the host process of the Windows service,0,windows_service,uptime,,

windows_service/tests/test_windows_service.py

@@ -42,6 +42,12 @@ def assert_service_check_and_metrics(aggregator, services):
             value=1,
             count=service.count,
         )
+        aggregator.assert_metric(
+            'windows_service.restarts',
+            tags=service.tags,
+            value=0,
+            count=service.count,
+        )
 
 
 def test_bad_config(check, instance_bad_config):
@@ -325,6 +331,64 @@ def test_name_regex_order(aggregator, check, instance_name_regex_prefix):
     assert_service_check_and_metrics(aggregator, services)
 
 
+def test_service_restart_detection(aggregator, check, instance_basic):
+    """
+    Test that service restarts are detected when the service PID changes between checks.
+    """
+    c = check(instance_basic)
+
+    mock_services = [
+        {
+            'ServiceName': 'EventLog',
+            'DisplayName': 'Windows Event Log',
+            'CurrentState': win32service.SERVICE_RUNNING,
+            'ProcessId': 1234,
+        },
+        {
+            'ServiceName': 'Dnscache',
+            'DisplayName': 'DNS Client',
+            'CurrentState': win32service.SERVICE_RUNNING,
+            'ProcessId': 5678,
+        },
+    ]
+
+    with patch('win32service.EnumServicesStatusEx', return_value=mock_services):
+        c.check(instance_basic)
+
+    # On first check, restarts should be 0
+    aggregator.assert_metric(
+        'windows_service.restarts',
+        value=0,
+        tags=['windows_service:EventLog', 'windows_service_state:running', 'service:EventLog', 'optional:tag1'],
+    )
+    aggregator.assert_metric(
+        'windows_service.restarts',
+        value=0,
+        tags=['windows_service:Dnscache', 'windows_service_state:running', 'service:Dnscache', 'optional:tag1'],
+    )
+
+    aggregator.reset()
+
+    # Only change the PID of EventLog
+    mock_services[0]['ProcessId'] = 9999
+
+    with patch('win32service.EnumServicesStatusEx', return_value=mock_services):
+        c.check(instance_basic)
+
+    # On second check, EventLog should have restarts=1
+    aggregator.assert_metric(
+        'windows_service.restarts',
+        value=1,
+        tags=['windows_service:EventLog', 'windows_service_state:running', 'service:EventLog', 'optional:tag1'],
+    )
+    # Dnscache should still have restarts=0
+    aggregator.assert_metric(
+        'windows_service.restarts',
+        value=0,
+        tags=['windows_service:Dnscache', 'windows_service_state:running', 'service:Dnscache', 'optional:tag1'],
+    )
+
+
 @pytest.mark.e2e
 def test_basic_e2e(dd_agent_check, check, instance_basic):
     aggregator = dd_agent_check(instance_basic)