New implementation of ProfilerDisabler: iteration 1

Introduce should_stop_sampling and should_stop_profiling.

should_stop_sampling is called whenever we sample while should_stop_profiling
is only called after we refresh config or submit profile.

Author: Colman Yau

codeguru_profiler_agent/local_aggregator.py

@@ -44,7 +44,7 @@ def __init__(self, reporter, environment=dict()):
         self.memory_limit_bytes = environment["memory_limit_bytes"]
         self.last_report_attempted = current_milli_time(clock=self.clock)
 
-        self._reset()
+        self.reset()
 
     def add(self, sample):
         """
@@ -69,7 +69,7 @@ def _check_memory_limit(self):
                     "Profiler memory usage limit has been reached")
             self.flush(force=True)
 
-    def _reset(self):
+    def reset(self):
         self.profile = self.profile_factory(
             profiling_group_name=self.profiling_group_name,
             sampling_interval_seconds=AgentConfiguration.get().sampling_interval.total_seconds(),
@@ -80,7 +80,7 @@ def _reset(self):
         self.timer.reset()
 
     @with_timer("flush")
-    def flush(self, force=False):
+    def flush(self, force=False, reset=True):
         now = current_milli_time(clock=self.clock)
         reported = False
         if not force and not self._is_over_reporting_interval(now):
@@ -92,8 +92,8 @@ def flush(self, force=False):
             self._report_profile(now)
             reported = True
 
-        if force or reported:
-            self._reset()
+        if force or (reset and reported):
+            self.reset()
         return reported
 
     def refresh_configuration(self):

codeguru_profiler_agent/profiler_disabler.py

@@ -20,11 +20,16 @@ def __init__(self, environment, clock=time.time):
         self.killswitch = KillSwitch(environment['killswitch_filepath'], clock)
         self.memory_limit_bytes = environment['memory_limit_bytes']
 
-    def should_stop_profiling(self, profile=None):
+    def should_stop_sampling(self, profile=None):
         return (self.killswitch.is_killswitch_on()
                 or self.cpu_usage_check.is_cpu_usage_limit_reached(profile)
                 or profile is not None and self._is_memory_limit_reached(profile))
 
+    def should_stop_profiling(self, profile=None):
+        return self.killswitch.is_killswitch_on() or \
+               (profile is not None and self.cpu_usage_check.is_overall_cpu_limit_reached(profile)
+                and self._is_memory_limit_reached(profile))
+
     def _is_memory_limit_reached(self, profile):
         return profile.get_memory_usage_bytes() > self.memory_limit_bytes
 
@@ -38,9 +43,29 @@ class CpuUsageCheck:
     def __init__(self, timer):
         self.timer = timer
 
+    # This function carries out an overall cpu limit check that covers the cpu overhead caused for the full
+    # sampling cycle: sample -> aggregate -> report -> refresh config. This has to be called with a profile
+    # which captured the total cycle cpu time usage. hnhg
+    def is_overall_cpu_limit_reached(self, profile):
+        profiler_metric = self.timer.metrics.get("runProfiler")
+        if not profile or not profiler_metric or profiler_metric.counter < MINIMUM_MEASURES_IN_DURATION_METRICS:
+            return False
+
+        used_time_percentage = 100 * profiler_metric.total/profile.get_active_millis_since_start()
+
+        if used_time_percentage >= AgentConfiguration.get().cpu_limit_percentage:
+            logger.debug(self.timer.metrics)
+            logger.info(
+                "Profiler cpu usage limit reached: {:.2f} % (limit: {:.2f} %), will stop CodeGuru Profiler.".format(
+                    used_time_percentage, AgentConfiguration.get().cpu_limit_percentage))
+            return True
+        else:
+            return False
+
     def is_cpu_usage_limit_reached(self, profile=None):
         sample_and_aggregate_metric = self.timer.metrics.get("sampleAndAggregate")
-        if not sample_and_aggregate_metric or sample_and_aggregate_metric.counter < MINIMUM_MEASURES_IN_DURATION_METRICS:
+        if not sample_and_aggregate_metric or \
+                sample_and_aggregate_metric.counter < MINIMUM_MEASURES_IN_DURATION_METRICS:
             return False
 
         sampling_interval_seconds = self._get_average_sampling_interval_seconds(profile)

codeguru_profiler_agent/profiler_runner.py

@@ -51,7 +51,7 @@ def start(self):
 
         :return: True if the profiler was started successfully; False otherwise.
         """
-        if self.profiler_disabler.should_stop_profiling():
+        if self.profiler_disabler.should_stop_sampling():
             logger.info("Profiler will not start.")
             return False
         self.scheduler.start()
@@ -71,26 +71,35 @@ def _profiling_command(self):
             if self._first_execution:
                 self.collector.setup()
                 self._first_execution = False
-            return self._run_profiler()
+            sample_result = self._run_profiler()
+            if sample_result.success and sample_result.should_check_overall:
+                if self.profiler_disabler.should_stop_profiling(profile=self.collector.profile):
+                    return False
+                if sample_result.should_reset:
+                    self.collector.reset()
+                    return True
+            return sample_result.success
         except:
             logger.info("An unexpected issue caused the profiling command to terminate.", exc_info=True)
             return False
 
     @with_timer("runProfiler")
     def _run_profiler(self):
-        if self.profiler_disabler.should_stop_profiling(self.collector.profile):
-            return False
+        if self.profiler_disabler.should_stop_sampling(self.collector.profile):
+            return RunProfilerStatus(success=False, should_check_overall=False, should_reset=False)
 
+        refreshed_config = False
         if not self.is_profiling_in_progress:
             self._refresh_configuration()
+            refreshed_config = True
 
         # after the refresh we may be working on a profile
         if self.is_profiling_in_progress:
-            if self.collector.flush():
+            if self.collector.flush(reset=False):
                 self.is_profiling_in_progress = False
-                return True
+                return RunProfilerStatus(success=True, should_check_overall=True, should_reset=True)
             self._sample_and_aggregate()
-        return True
+        return RunProfilerStatus(success=True, should_check_overall=refreshed_config, should_reset=False)
 
     @with_timer("sampleAndAggregate")
     def _sample_and_aggregate(self):
@@ -129,3 +138,10 @@ def pause(self, block=False):
         """
         self.scheduler.pause(block)
         self.collector.profile.pause()
+
+
+class RunProfilerStatus:
+    def __init__(self, success, should_check_overall, should_reset):
+        self.success = success
+        self.should_check_overall = should_check_overall
+        self.should_reset = should_reset

test/unit/test_profiler_disabler.py

@@ -72,15 +72,15 @@ def before(self):
 
     def test_it_stops_profiling_if_killswitch_is_on(self):
         self.disabler.killswitch.is_killswitch_on = Mock(return_value=True)
-        assert self.disabler.should_stop_profiling(self.profiler)
+        assert self.disabler.should_stop_sampling(self.profiler)
 
     def test_it_stops_profiling_if_memory_limit_is_reached(self):
         self.disabler._is_memory_limit_reached = Mock(return_value=True)
-        assert self.disabler.should_stop_profiling(self.profiler)
+        assert self.disabler.should_stop_sampling(self.profiler)
 
     def test_it_stops_profiling_if_process_duration_is_reached(self):
         self.disabler.cpu_usage_check.is_cpu_usage_limit_reached = Mock(return_value=True)
-        assert self.disabler.should_stop_profiling(self.profiler)
+        assert self.disabler.should_stop_sampling(self.profiler)
 
 
 class TestKillSwitch: