TEST: Preserve LeakReplay end-to-end coverage

tests/end_to_end/test_scenarios.py

@@ -74,6 +74,7 @@
 SCENARIO_MAX_TOTAL: dict[str, int] = {
     "airt.psychosocial": 2,  # One objective per sub-harm.
     "garak.latent_injection": 23,  # One objective per family/trigger pair.
+    "garak.leak_replay": 8,  # One objective per technique and corpus.
     "garak.prompt_inject": 3,  # One objective per goal text.
     "garak.web_injection": 6,  # One objective per technique-specific population.
 }

tests/unit/scenario/test_scenario_e2e_contract.py

@@ -10,6 +10,7 @@
 
 from pyrit.cli.pyrit_scan import _build_run_request, parse_args
 from pyrit.registry import ScenarioRegistry
+from pyrit.scenario.garak import LeakReplay, LeakReplayDatasetConfiguration
 from pyrit.scenario.scenarios.airt import Psychosocial
 from pyrit.score import SubStringScorer
 
@@ -44,6 +45,27 @@ def test_e2e_launch_uses_bounded_supported_dataset_flags(
         assert request.datasets.max_total == expected_max_total
 
 
+@pytest.mark.usefixtures("patch_central_database")
+async def test_leak_replay_e2e_launch_covers_default_populations_async(capsys: pytest.CaptureFixture[str]) -> None:
+    scenario_name = "garak.leak_replay"
+    with patch.object(scenario_tests, "pyrit_scan_main", return_value=0) as run:
+        scenario_tests.test_scenario_with_pyrit_scan(scenario_name=scenario_name, capsys=capsys)
+    run.assert_called_once()
+    request = _build_run_request(parsed_args=parse_args(run.call_args.args[0]), scenario_name=scenario_name)
+    assert request.datasets is not None
+
+    scenario = LeakReplay()
+    scenario.set_params_from_args(args={"dataset_overrides": request.datasets.model_dump(exclude_none=True)})
+    await scenario.get_run_size_estimate_async()
+    assert isinstance(scenario._dataset_config, LeakReplayDatasetConfiguration)
+    populations = scenario._dataset_config.population_names()
+    assert set(populations) == {
+        f"{technique}_{corpus}"
+        for technique in ("cloze", "completion")
+        for corpus in ("literature", "potter", "nyt", "guardian")
+    }
+
+
 @pytest.mark.usefixtures("patch_central_database")
 @pytest.mark.parametrize("scenario_name", list(scenario_tests.SCENARIO_MAX_TOTAL))
 async def test_e2e_coverage_limits_pass_real_scenario_validation_async(scenario_name: str) -> None: