Merge pull request #1134 from cloudbees-oss/rename-flake

Minor improvements on the early flake detection

Author: ono-max

launchable/__main__.py

@@ -10,9 +10,9 @@
 from launchable.app import Application
 
 from .commands.compare import compare
+from .commands.detect_flakes import detect_flakes
 from .commands.inspect import inspect
 from .commands.record import record
-from .commands.retry import retry
 from .commands.split_subset import split_subset
 from .commands.stats import stats
 from .commands.subset import subset
@@ -92,7 +92,7 @@ def main(ctx, log_level, plugin_dir, dry_run, skip_cert_verification):
 main.add_command(inspect)
 main.add_command(stats)
 main.add_command(compare)
-main.add_command(retry)
+main.add_command(detect_flakes, "detect-flakes")
 
 if __name__ == '__main__':
     main()

…chable/commands/retry/flake_detection.py launchable/commands/detect_flakes.pylaunchable/commands/retry/flake_detection.py renamed to launchable/commands/detect_flakes.py launchable/commands/retry/flake_detection.py renamed to launchable/commands/detect_flakes.py

@@ -6,12 +6,13 @@
 from launchable.app import Application
 from launchable.commands.helper import find_or_create_session
 from launchable.commands.test_path_writer import TestPathWriter
+from launchable.testpath import unparse_test_path
 from launchable.utils.click import ignorable_error
 from launchable.utils.env_keys import REPORT_ERROR_KEY
 from launchable.utils.launchable_client import LaunchableClient
 from launchable.utils.tracking import Tracking, TrackingClient
 
-from ...utils.commands import Command
+from ..utils.commands import Command
 
 
 @click.group(help="Early flake detection")
@@ -23,14 +24,16 @@
     required=True
 )
 @click.option(
-    '--confidence',
-    help='Confidence level for flake detection',
+    '--retry-threshold',
+    'retry_threshold',
+    help='Throughness of how "flake" is detected',
     type=click.Choice(['low', 'medium', 'high'], case_sensitive=False),
+    default='medium',
     required=True,
 )
 @click.pass_context
-def flake_detection(ctx, confidence, session):
-    tracking_client = TrackingClient(Command.FLAKE_DETECTION, app=ctx.obj)
+def detect_flakes(ctx, retry_threshold, session):
+    tracking_client = TrackingClient(Command.DETECT_FLAKE, app=ctx.obj)
     client = LaunchableClient(app=ctx.obj, tracking_client=tracking_client, test_runner=ctx.invoked_subcommand)
     session_id = None
     try:
@@ -64,15 +67,18 @@ def run(self):
             try:
                 res = client.request(
                     "get",
-                    "retry/flake-detection",
+                    "detect-flake",
                     params={
-                        "confidence": confidence.upper(),
+                        "confidence": retry_threshold.upper(),
                         "session-id": os.path.basename(session_id),
                         "test-runner": ctx.invoked_subcommand})
                 res.raise_for_status()
                 test_paths = res.json().get("testPaths", [])
                 if test_paths:
                     self.print(test_paths)
+                    click.echo("Trying to retry the following tests:", err=True)
+                    for detail in res.json().get("testDetails", []):
+                        click.echo(f"{detail.get('reason')}: {unparse_test_path(detail.get('fullTestPath'))}", err=True)
             except Exception as e:
                 tracking_client.send_error_event(
                     event_name=Tracking.ErrorEvent.INTERNAL_CLI_ERROR,

launchable/commands/retry/__init__.py

@@ -33,7 +33,7 @@ def subset(client):
 split_subset = launchable.CommonSplitSubsetImpls(__name__,
                                                  formatter=lambda x: x[0]['name'] + ":" + x[1]['name']).split_subset()
 
-launchable.CommonFlakeDetectionImpls(__name__, formatter=lambda x: x[0]['name'] + ":" + x[1]['name']).flake_detection()
+launchable.CommonFlakeDetectionImpls(__name__, formatter=lambda x: x[0]['name'] + ":" + x[1]['name']).detect_flakes()
 
 
 @click.argument('workspace', required=True)

launchable/test_runners/bazel.py

@@ -53,4 +53,4 @@ def find_filename():
 
 split_subset = launchable.CommonSplitSubsetImpls(__name__).split_subset()
 
-launchable.CommonFlakeDetectionImpls(__name__).flake_detection()
+launchable.CommonFlakeDetectionImpls(__name__).detect_flakes()

launchable/test_runners/file.py

@@ -5,11 +5,10 @@
 
 import click
 
+from launchable.commands.detect_flakes import detect_flakes as detect_flakes_cmd
 from launchable.commands.record.tests import tests as record_tests_cmd
-from launchable.commands.retry.flake_detection import flake_detection as flake_detection_cmd
 from launchable.commands.split_subset import split_subset as split_subset_cmd
 from launchable.commands.subset import subset as subset_cmd
-from launchable.testpath import unparse_test_path
 
 
 def cmdname(m):
@@ -46,7 +45,7 @@ def subset(f):
 
 
 def flake_detection(f):
-    return wrap(f, flake_detection_cmd)
+    return wrap(f, detect_flakes_cmd)
 
 
 def split_subset(f):
@@ -177,15 +176,15 @@ class CommonFlakeDetectionImpls:
     def __init__(
             self,
             module_name,
-            formatter=unparse_test_path,
+            formatter=None,
             seperator="\n",
     ):
         self.cmdname = cmdname(module_name)
         self._formatter = formatter
         self._separator = seperator
 
-    def flake_detection(self):
-        def flake_detection(client):
+    def detect_flakes(self):
+        def detect_flakes(client):
             if self._formatter:
                 client.formatter = self._formatter
 
@@ -194,4 +193,4 @@ def flake_detection(client):
 
             client.run()
 
-        return wrap(flake_detection, flake_detection_cmd, self.cmdname)
+        return wrap(detect_flakes, detect_flakes_cmd, self.cmdname)

launchable/test_runners/launchable.py

@@ -47,7 +47,7 @@ def subset(client, test_path_file):
 
 split_subset = launchable.CommonSplitSubsetImpls(__name__, formatter=unparse_test_path, seperator='\n').split_subset()
 
-launchable.CommonFlakeDetectionImpls(__name__).flake_detection()
+launchable.CommonFlakeDetectionImpls(__name__).detect_flakes()
 
 
 @click.argument('test_result_files', required=True, type=click.Path(exists=True), nargs=-1)

launchable/test_runners/raw.py

@@ -3,3 +3,4 @@
 subset = launchable.CommonSubsetImpls(__name__).scan_files('*_spec.rb')
 split_subset = launchable.CommonSplitSubsetImpls(__name__).split_subset()
 record_tests = launchable.CommonRecordTestImpls(__name__).report_files()
+launchable.CommonFlakeDetectionImpls(__name__).detect_flakes()

launchable/test_runners/rspec.py

@@ -8,7 +8,7 @@ class Command(Enum):
     RECORD_SESSION = 'RECORD_SESSION'
     SUBSET = 'SUBSET'
     COMMIT = 'COMMIT'
-    FLAKE_DETECTION = 'FLAKE_DETECTION'
+    DETECT_FLAKE = 'DETECT_FLAKE'
 
     def display_name(self):
         return self.value.lower().replace('_', ' ')

launchable/utils/commands.py

@@ -7,7 +7,7 @@
 from tests.cli_test_case import CliTestCase
 
 
-class FlakeDetectionTest(CliTestCase):
+class DetectFlakeTest(CliTestCase):
     @responses.activate
     @mock.patch.dict(os.environ, {"LAUNCHABLE_TOKEN": CliTestCase.launchable_token})
     def test_flake_detection_success(self):
@@ -19,16 +19,15 @@ def test_flake_detection_success(self):
         }
         responses.add(
             responses.GET,
-            f"{get_base_url()}/intake/organizations/{self.organization}/workspaces/{self.workspace}/retry/flake-detection",
+            f"{get_base_url()}/intake/organizations/{self.organization}/workspaces/{self.workspace}/detect-flake",
             json=mock_json_response,
             status=200,
         )
         result = self.cli(
-            "retry",
-            "flake-detection",
+            "detect-flakes",
             "--session",
             self.session,
-            "--confidence",
+            "--retry-threshold",
             "high",
             "file",
             mix_stderr=False,
@@ -37,22 +36,47 @@ def test_flake_detection_success(self):
         self.assertIn("test_flaky_1.py", result.stdout)
         self.assertIn("test_flaky_2.py", result.stdout)
 
+    @responses.activate
+    @mock.patch.dict(os.environ, {"LAUNCHABLE_TOKEN": CliTestCase.launchable_token})
+    def test_flake_detection_without_retry_threshold_success(self):
+        mock_json_response = {
+            "testPaths": [
+                [{"type": "file", "name": "test_flaky_1.py"}],
+                [{"type": "file", "name": "test_flaky_2.py"}],
+            ]
+        }
+        responses.add(
+            responses.GET,
+            f"{get_base_url()}/intake/organizations/{self.organization}/workspaces/{self.workspace}/detect-flake",
+            json=mock_json_response,
+            status=200,
+        )
+        result = self.cli(
+            "detect-flakes",
+            "--session",
+            self.session,
+            "file",
+            mix_stderr=False,
+        )
+        self.assert_success(result)
+        self.assertIn("test_flaky_1.py", result.stdout)
+        self.assertIn("test_flaky_2.py", result.stdout)
+
     @responses.activate
     @mock.patch.dict(os.environ, {"LAUNCHABLE_TOKEN": CliTestCase.launchable_token})
     def test_flake_detection_no_flakes(self):
         mock_json_response = {"testPaths": []}
         responses.add(
             responses.GET,
-            f"{get_base_url()}/intake/organizations/{self.organization}/workspaces/{self.workspace}/retry/flake-detection",
+            f"{get_base_url()}/intake/organizations/{self.organization}/workspaces/{self.workspace}/detect-flake",
             json=mock_json_response,
             status=200,
         )
         result = self.cli(
-            "retry",
-            "flake-detection",
+            "detect-flakes",
             "--session",
             self.session,
-            "--confidence",
+            "--retry-threshold",
             "low",
             "file",
             mix_stderr=False,
@@ -65,15 +89,14 @@ def test_flake_detection_no_flakes(self):
     def test_flake_detection_api_error(self):
         responses.add(
             responses.GET,
-            f"{get_base_url()}/intake/organizations/{self.organization}/workspaces/{self.workspace}/retry/flake-detection",
+            f"{get_base_url()}/intake/organizations/{self.organization}/workspaces/{self.workspace}/detect-flake",
             status=500,
         )
         result = self.cli(
-            "retry",
-            "flake-detection",
+            "detect-flakes",
             "--session",
             self.session,
-            "--confidence",
+            "--retry-threshold",
             "medium",
             "file",
             mix_stderr=False,