diff --git a/tests/end_to_end/test_scenarios.py b/tests/end_to_end/test_scenarios.py index de3181b5d6..7f6b2270dd 100644 --- a/tests/end_to_end/test_scenarios.py +++ b/tests/end_to_end/test_scenarios.py @@ -74,6 +74,7 @@ SCENARIO_MAX_TOTAL: dict[str, int] = { "airt.psychosocial": 2, # One objective per sub-harm. "garak.latent_injection": 23, # One objective per family/trigger pair. + "garak.leak_replay": 8, # One objective per technique and corpus. "garak.prompt_inject": 3, # One objective per goal text. "garak.web_injection": 6, # One objective per technique-specific population. } diff --git a/tests/unit/scenario/test_scenario_e2e_contract.py b/tests/unit/scenario/test_scenario_e2e_contract.py index 2c1475f5bb..417d985e8c 100644 --- a/tests/unit/scenario/test_scenario_e2e_contract.py +++ b/tests/unit/scenario/test_scenario_e2e_contract.py @@ -10,6 +10,7 @@ from pyrit.cli.pyrit_scan import _build_run_request, parse_args from pyrit.registry import ScenarioRegistry +from pyrit.scenario.garak import LeakReplay, LeakReplayDatasetConfiguration from pyrit.scenario.scenarios.airt import Psychosocial from pyrit.score import SubStringScorer @@ -44,6 +45,27 @@ def test_e2e_launch_uses_bounded_supported_dataset_flags( assert request.datasets.max_total == expected_max_total +@pytest.mark.usefixtures("patch_central_database") +async def test_leak_replay_e2e_launch_covers_default_populations_async(capsys: pytest.CaptureFixture[str]) -> None: + scenario_name = "garak.leak_replay" + with patch.object(scenario_tests, "pyrit_scan_main", return_value=0) as run: + scenario_tests.test_scenario_with_pyrit_scan(scenario_name=scenario_name, capsys=capsys) + run.assert_called_once() + request = _build_run_request(parsed_args=parse_args(run.call_args.args[0]), scenario_name=scenario_name) + assert request.datasets is not None + + scenario = LeakReplay() + scenario.set_params_from_args(args={"dataset_overrides": request.datasets.model_dump(exclude_none=True)}) + await scenario.get_run_size_estimate_async() + assert isinstance(scenario._dataset_config, LeakReplayDatasetConfiguration) + populations = scenario._dataset_config.population_names() + assert set(populations) == { + f"{technique}_{corpus}" + for technique in ("cloze", "completion") + for corpus in ("literature", "potter", "nyt", "guardian") + } + + @pytest.mark.usefixtures("patch_central_database") @pytest.mark.parametrize("scenario_name", list(scenario_tests.SCENARIO_MAX_TOTAL)) async def test_e2e_coverage_limits_pass_real_scenario_validation_async(scenario_name: str) -> None: