feat: add a --seed option to the test command

Author: azmeuk

doc/changelog.rst

@@ -8,6 +8,7 @@ Added
 ^^^^^
 - Support for Python 3.15.
 - ``test --strict`` fails on deviations from the recommendations of the RFCs.
+- ``test --seed`` sends the same generated values as a previous run. Each run prints its seed.
 
 Changed
 ^^^^^^^

doc/tutorial.rst

@@ -257,13 +257,22 @@ See the :doc:`scim2-tester documentation <scim2_tester:index>` for more details
     ...
     SUCCESS    check_replace_attribute User displayName
     ...
+    Seed: 2416347125
 
 Each result shows the status of the check, its title, and the resource type and the attribute it is about.
 The passing results are green, the deviations from the recommendations of the RFCs are yellow,
 the failures are red and the skipped checks are gray.
 The command fails with the exit code 1 when a check fails.
 With ``--strict``, it also fails on deviations.
 
+The checks send generated values, such as user names and email addresses.
+Each run generates new values, and prints its seed at the end.
+To send the same values again, for instance to replay a failure, pass the seed with ``--seed``:
+
+.. code-block:: console
+
+    $ scim test --seed 2416347125
+
 Errors
 ------
 

scim2_cli/test.py

@@ -1,3 +1,4 @@
+import secrets
 import sys
 
 import click
@@ -39,12 +40,18 @@
     is_flag=True,
     help="Fail on deviations from the recommendations of the RFCs.",
 )
+@click.option(
+    "--seed",
+    type=int,
+    help="Send the same generated values as the run that printed this seed.",
+)
 def test_cli(
     ctx: click.Context,
     verbose: bool,
     check_status_code: bool,
     check_content_type: bool,
     strict: bool,
+    seed: int | None,
 ) -> None:
     """Perform a server SCIM compliance check using :doc:`scim2-tester <scim2_tester:index>`.
 
@@ -55,7 +62,9 @@ def test_cli(
     client = ctx.obj.client
     client.check_response_status_codes = check_status_code
     client.check_response_content_type = check_content_type
-    results = check_server(client)
+    if seed is None:
+        seed = secrets.randbelow(2**32)
+    results = check_server(client, seed=seed)
     click.echo(f"Performing a SCIM compliance check on {client.client.base_url} ...")
     failed = False
     for result in results:
@@ -77,4 +86,5 @@ def test_cli(
             if verbose and result.data:
                 click.echo(f"  {escape_control_characters(result.data)}")
 
+    click.echo(f"Seed: {seed}")
     sys.exit(1 if failed else 0)

tests/test_test.py

@@ -26,17 +26,19 @@ def test_nominal(runner, httpserver):
             data="data2",
         ),
     ]
-    with patch("scim2_cli.test.check_server", side_effect=[results]):
+    with patch("scim2_cli.test.check_server", side_effect=[results]) as check_server:
         result = runner.invoke(
             cli,
             ["--url", httpserver.url_for("/"), "test"],
             catch_exceptions=False,
         )
+        seed = check_server.call_args.kwargs["seed"]
 
         expected_output = f"""Performing a SCIM compliance check on http://localhost:{httpserver.port}/ ...
 SUCCESS    test1
   reason1
 ERROR      test2
+Seed: {seed}
 """
         assert result.output == expected_output
 
@@ -59,12 +61,13 @@ def test_verbose(runner, httpserver):
             data="data2",
         ),
     ]
-    with patch("scim2_cli.test.check_server", side_effect=[results]):
+    with patch("scim2_cli.test.check_server", side_effect=[results]) as check_server:
         result = runner.invoke(
             cli,
             ["--url", httpserver.url_for("/"), "test", "--verbose"],
             catch_exceptions=False,
         )
+        seed = check_server.call_args.kwargs["seed"]
 
         expected_output = f"""Performing a SCIM compliance check on http://localhost:{httpserver.port}/ ...
 SUCCESS    test1
@@ -73,6 +76,7 @@ def test_verbose(runner, httpserver):
 ERROR      test2
   reason2
   data2
+Seed: {seed}
 """
         assert result.output == expected_output
 
@@ -97,7 +101,7 @@ def test_response_checks_options(runner, httpserver, options, expected):
     """Test that the response check options are applied on the client."""
     checked = {}
 
-    def check_server(client):
+    def check_server(client, seed):
         checked["status_codes"] = client.check_response_status_codes
         checked["content_type"] = client.check_response_content_type
         return []
@@ -140,7 +144,7 @@ def test_resource_type_and_attribute(runner, httpserver):
 
     result = run_checks(runner, httpserver, results)
 
-    assert result.output.splitlines()[1:] == [
+    assert result.output.splitlines()[1:-1] == [
         "SUCCESS    discovery_methods",
         "SUCCESS    object_creation User",
         "SUCCESS    check_replace_attribute User name.familyName",
@@ -209,3 +213,45 @@ def test_status_color_follows_the_outcome(runner, httpserver, status, color):
 def test_every_outcome_has_a_color():
     """Each outcome of scim2-tester has a color."""
     assert set(OUTCOME_COLORS) == set(Outcome)
+
+
+def test_seed_is_sent_and_printed(runner, httpserver):
+    """The seed passed with --seed generates the values, and is printed at the end."""
+    with patch("scim2_cli.test.check_server", return_value=[]) as check_server:
+        result = runner.invoke(
+            cli,
+            ["--url", httpserver.url_for("/"), "test", "--seed", "1234"],
+            catch_exceptions=False,
+        )
+
+    assert check_server.call_args.kwargs["seed"] == 1234
+    assert result.output.splitlines()[-1] == "Seed: 1234"
+
+
+def test_each_run_has_its_own_seed(runner, httpserver):
+    """Without --seed, each run draws a new seed and prints it."""
+    seeds = []
+    for _ in range(2):
+        with patch("scim2_cli.test.check_server", return_value=[]) as check_server:
+            result = runner.invoke(
+                cli,
+                ["--url", httpserver.url_for("/"), "test"],
+                catch_exceptions=False,
+            )
+        seed = check_server.call_args.kwargs["seed"]
+        assert result.output.splitlines()[-1] == f"Seed: {seed}"
+        seeds.append(seed)
+
+    assert seeds[0] != seeds[1]
+
+
+def test_seed_must_be_an_integer(runner, httpserver):
+    """A seed that is not an integer is a usage error."""
+    with patch("scim2_cli.test.check_server") as check_server:
+        result = runner.invoke(
+            cli, ["--url", httpserver.url_for("/"), "test", "--seed", "abc"]
+        )
+
+    assert result.exit_code == 2
+    assert "'abc' is not a valid integer" in result.output
+    check_server.assert_not_called()