feat: print each result of the test command as soon as its check ends

Author: azmeuk

doc/changelog.rst

@@ -1,6 +1,14 @@
 Changelog
 =========
 
+[0.7.1] - Unreleased
+--------------------
+
+Changed
+^^^^^^^
+- The ``test`` command prints each result as soon as its check ends, instead of at the end of
+  the run. Requires scim2-tester 0.6.4, which also brings new checks.
+
 [0.7.0] - 2026-10-29
 --------------------
 

pyproject.toml

@@ -29,7 +29,7 @@ requires-python = ">= 3.11"
 dependencies = [
     "click>=8.5.0",
     "scim2-client[httpx2]>=0.14.0",
-    "scim2-tester>=0.6.3",
+    "scim2-tester>=0.6.4",
     "sphinx-click-rst-to-ansi-formatter>=0.2.0",
     "pydanclick>=0.5.0",
     "pygments>=2.18.0",

scim2_cli/test.py

@@ -5,7 +5,7 @@
 import click
 from scim2_tester import Outcome
 from scim2_tester import Status
-from scim2_tester import check_server
+from scim2_tester import iter_check_server
 from sphinx_click.rst_to_ansi_formatter import make_rst_to_ansi_formatter
 
 from .utils import DOC_URL
@@ -89,7 +89,7 @@ def test_cli(
     if seed is None:
         seed = secrets.randbelow(2**32)
     click.echo(f"Performing a SCIM compliance check on {client.client.base_url} ...")
-    results = check_server(client, seed=seed)
+    results = iter_check_server(client, seed=seed)
     failed = False
     outcomes: Counter[Outcome] = Counter()
     for result in results:

tests/test_escape.py

@@ -77,7 +77,7 @@ def test_compliance_report_is_escaped(runner, httpserver, field, sequence, escap
     )
     setattr(result, field, f"{getattr(result, field)}{sequence}")
 
-    with patch("scim2_cli.test.check_server", side_effect=[[result]]):
+    with patch("scim2_cli.test.iter_check_server", side_effect=[[result]]):
         output = runner.invoke(
             cli, ["--url", httpserver.url_for("/"), "test", "-vv"], color=True
         ).output
@@ -146,7 +146,7 @@ def test_line_breaks_and_tabulations_are_kept(runner, httpserver):
     """Line breaks and tabulations are the layout of the messages, not escape sequences."""
     result = CheckResult(status=Status.ERROR, title="check", reason="first\n\tsecond")
 
-    with patch("scim2_cli.test.check_server", side_effect=[[result]]):
+    with patch("scim2_cli.test.iter_check_server", side_effect=[[result]]):
         output = runner.invoke(
             cli, ["--url", httpserver.url_for("/"), "test"], color=True
         ).output

tests/test_test.py

@@ -11,7 +11,7 @@
 
 
 def run_checks(runner, httpserver, results, *options, color=False):
-    with patch("scim2_cli.test.check_server", side_effect=[results]):
+    with patch("scim2_cli.test.iter_check_server", side_effect=[results]):
         return runner.invoke(
             cli,
             ["--url", httpserver.url_for("/"), "test", *options],
@@ -112,12 +112,12 @@ def test_response_checks_options(runner, httpserver, options, expected):
     """Test that the response check options are applied on the client."""
     checked = {}
 
-    def check_server(client, seed):
+    def iter_check_server(client, seed):
         checked["status_codes"] = client.check_response_status_codes
         checked["content_type"] = client.check_response_content_type
         return []
 
-    with patch("scim2_cli.test.check_server", check_server):
+    with patch("scim2_cli.test.iter_check_server", iter_check_server):
         result = runner.invoke(
             cli,
             ["--url", httpserver.url_for("/"), "test", *options],
@@ -218,28 +218,32 @@ def test_every_outcome_has_a_color():
 
 def test_seed_is_sent_and_printed(runner, httpserver):
     """The seed passed with --seed generates the values, and is printed at the end."""
-    with patch("scim2_cli.test.check_server", return_value=[]) as check_server:
+    with patch(
+        "scim2_cli.test.iter_check_server", return_value=[]
+    ) as iter_check_server:
         result = runner.invoke(
             cli,
             ["--url", httpserver.url_for("/"), "test", "--seed", "1234"],
             catch_exceptions=False,
         )
 
-    assert check_server.call_args.kwargs["seed"] == 1234
+    assert iter_check_server.call_args.kwargs["seed"] == 1234
     assert result.output.splitlines()[-1] == "no checks ran (seed 1234)"
 
 
 def test_each_run_has_its_own_seed(runner, httpserver):
     """Without --seed, each run draws a new seed and prints it."""
     seeds = []
     for _ in range(2):
-        with patch("scim2_cli.test.check_server", return_value=[]) as check_server:
+        with patch(
+            "scim2_cli.test.iter_check_server", return_value=[]
+        ) as iter_check_server:
             result = runner.invoke(
                 cli,
                 ["--url", httpserver.url_for("/"), "test"],
                 catch_exceptions=False,
             )
-        seed = check_server.call_args.kwargs["seed"]
+        seed = iter_check_server.call_args.kwargs["seed"]
         assert result.output.splitlines()[-1] == f"no checks ran (seed {seed})"
         seeds.append(seed)
 
@@ -248,24 +252,24 @@ def test_each_run_has_its_own_seed(runner, httpserver):
 
 def test_seed_must_be_an_integer(runner, httpserver):
     """A seed that is not an integer is a usage error."""
-    with patch("scim2_cli.test.check_server") as check_server:
+    with patch("scim2_cli.test.iter_check_server") as iter_check_server:
         result = runner.invoke(
             cli, ["--url", httpserver.url_for("/"), "test", "--seed", "abc"]
         )
 
     assert result.exit_code == 2
     assert "'abc' is not a valid integer" in result.output
-    check_server.assert_not_called()
+    iter_check_server.assert_not_called()
 
 
 def test_header_is_printed_before_the_checks(runner, httpserver):
     """The header is printed before the checks run."""
 
-    def check_server(client, seed):
+    def iter_check_server(client, seed):
         click.echo("checks running")
         return []
 
-    with patch("scim2_cli.test.check_server", check_server):
+    with patch("scim2_cli.test.iter_check_server", iter_check_server):
         result = runner.invoke(
             cli, ["--url", httpserver.url_for("/"), "test"], catch_exceptions=False
         )
@@ -276,6 +280,26 @@ def check_server(client, seed):
     ]
 
 
+def test_results_are_printed_as_soon_as_they_arrive(runner, httpserver):
+    """Each result is printed before the next check runs."""
+
+    def iter_check_server(client, seed):
+        yield CheckResult(status=Status.ERROR, title="test1")
+        click.echo("next check running")
+        yield CheckResult(status=Status.ERROR, title="test2")
+
+    with patch("scim2_cli.test.iter_check_server", iter_check_server):
+        result = runner.invoke(
+            cli, ["--url", httpserver.url_for("/"), "test"], catch_exceptions=False
+        )
+
+    assert result.output.splitlines()[1:4] == [
+        "ERROR      test1",
+        "next check running",
+        "ERROR      test2",
+    ]
+
+
 @pytest.mark.parametrize(
     ("statuses", "expected"),
     [