From 196adae0ac3ff372e64012c6f75abe2f90ad31f0 Mon Sep 17 00:00:00 2001 From: Richard Lundeen Date: Tue, 22 Sep 2026 16:50:58 -0700 Subject: [PATCH 1/2] FIX: Make commutative scorer evaluation identity order-independent Opt built-in AND, OR, and MAJORITY composites into conditional child-hash sorting. Preserve duplicates, execution order, and ordered custom aggregates. Keep legacy identifiers fail-closed and document resume compatibility. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- doc/code/memory/3_memory_data_types.md | 22 ++++ .../identifiers/evaluation_identifier.py | 12 +- .../models/identifiers/evaluation_markers.py | 8 +- pyrit/models/identifiers/scorer_identifier.py | 12 +- .../true_false/true_false_composite_scorer.py | 17 ++- .../identifiers/test_evaluation_identifier.py | 56 +++++++++ .../identifiers/test_evaluation_markers.py | 7 ++ tests/unit/scenario/core/test_scenario.py | 48 +++++++- .../test_scorer_evaluation_identifier.py | 107 +++++++++++++++++- 9 files changed, 280 insertions(+), 9 deletions(-) diff --git a/doc/code/memory/3_memory_data_types.md b/doc/code/memory/3_memory_data_types.md index 7ab05d53bf..4df2604524 100644 --- a/doc/code/memory/3_memory_data_types.md +++ b/doc/code/memory/3_memory_data_types.md @@ -165,3 +165,25 @@ What feeds the eval hash is declared **on the strongly-typed identifier fields t - `Evaluate.Unwrap()` — mark a wrapper passthrough slot (e.g. `TargetIdentifier.targets`). A multi-target like `RoundRobinTarget` is "looked through" to its inner target, so it eval-hashes the same as the bare inner target. For example, `TargetIdentifier` excludes `endpoint` but includes `temperature`, and the `ObjectiveTargetEvaluationIdentifier` / `ScorerEvaluationIdentifier` / `AtomicAttackEvaluationIdentifier` subclasses derive their engine rules from these markers (via `derive_eval_config`). Markers affect **only** the eval hash — the identity `hash` always keeps distinct components (e.g. a wrapper vs. its inner target) distinct. + +#### Child order + +Child lists are ordered unless a child field declares +`Evaluate.Include(unordered_when="parent_param")` and that parent param is exactly +`True`. The eval engine then sorts the child hashes **after** behavioral filtering. +It retains duplicates and does not change the stored list or execution order. + +`TrueFalseCompositeScorer` sets the `sub_scorers_order_independent` param only for +the built-in `AND`, `OR`, and `MAJORITY` aggregator functions. Reordered children +then have the same eval hash, including within nested composites and scenario +resume checks. Different child configurations and duplicate counts remain distinct. +Custom aggregators, converter pipelines, and other sequences keep their order. +Rationale and metadata output can still follow input order. + +**Compatibility:** The new opt-in param changes the content and eval hashes of +built-in composites and identifiers that include them. Existing records without +the param remain ordered: an aggregator name alone cannot prove that a custom +function is commutative. No stored hashes are migrated. A scenario saved before +this opt-in cannot resume with a new built-in composite; start a new run. +Stored eval hashes are recomputed when wrapped in an `EvaluationIdentifier`, +but hash-only lookup keys and cached evaluation results are not remapped. diff --git a/pyrit/models/identifiers/evaluation_identifier.py b/pyrit/models/identifiers/evaluation_identifier.py index b34ac45f99..d80dcdca48 100644 --- a/pyrit/models/identifiers/evaluation_identifier.py +++ b/pyrit/models/identifiers/evaluation_identifier.py @@ -82,6 +82,9 @@ class ChildEvalRule(BaseModel): ``RoundRobinTarget``). The first item of that sub-child list is substituted before applying param filtering, so the eval hash matches the unwrapped inner target. ``None`` means no unwrapping. + * ``unordered_when`` — names a boolean param on the parent identifier. + Only an explicit ``True`` sorts this slot's projected child hashes. + Duplicates are retained, and the original child list is not changed. """ model_config = ConfigDict(frozen=True) @@ -91,6 +94,7 @@ class ChildEvalRule(BaseModel): included_item_values: dict[str, Any] | None = Field(default=None) param_fallbacks: dict[str, str] | None = Field(default=None) inner_child_name: str | None = Field(default=None) + unordered_when: str | None = None def _build_eval_dict( @@ -188,6 +192,8 @@ def _build_eval_dict( ) for c in child_list ] + if rule and rule.unordered_when and identifier.params.get(rule.unordered_when) is True: + hashes.sort() eval_children[name] = hashes[0] if len(hashes) == 1 else hashes if eval_children: eval_dict["children"] = eval_children @@ -227,7 +233,7 @@ def compute_eval_hash( own_rule (ChildEvalRule | None): Rule applied to the root entity's own params and fallbacks. Only ``included_params`` and ``param_fallbacks`` are honored; ``exclude``, ``included_item_values``, - and ``inner_child_name`` are not meaningful at the root and will + ``inner_child_name``, and ``unordered_when`` are not meaningful at the root and will raise ``ValueError`` if set. Defaults to None. root_unwrap_child (str | None): If set, names the wrapper passthrough slot on the root identifier (e.g. ``"targets"``). When the root is a @@ -249,6 +255,8 @@ def compute_eval_hash( raise ValueError("own_rule.included_item_values is not meaningful at the root entity") if own_rule.inner_child_name is not None: raise ValueError("own_rule.inner_child_name is not meaningful at the root entity") + if own_rule.unordered_when is not None: + raise ValueError("own_rule.unordered_when is not meaningful at the root entity") if root_unwrap_child is not None: inner = identifier.get_child_list(root_unwrap_child) @@ -310,6 +318,7 @@ def _slot_rule( included_params=included_params, param_fallbacks=fallbacks, inner_child_name=_type_unwrap_field(child_type), + unordered_when=marker.unordered_when if isinstance(marker, Include) else None, ) @@ -321,6 +330,7 @@ def _is_neutral_rule(rule: ChildEvalRule) -> bool: and rule.included_item_values is None and rule.param_fallbacks is None and rule.inner_child_name is None + and rule.unordered_when is None ) diff --git a/pyrit/models/identifiers/evaluation_markers.py b/pyrit/models/identifiers/evaluation_markers.py index 9b1ad1aca2..dccddd5436 100644 --- a/pyrit/models/identifiers/evaluation_markers.py +++ b/pyrit/models/identifiers/evaluation_markers.py @@ -29,7 +29,9 @@ class TargetIdentifier(ComponentIdentifier): * ``Evaluate.Include()`` — include the child, projecting its subtree with the child type's own markers (the default for an unmarked field). ``only_params`` overrides that projection for this slot, restricting the child subtree to the - named params (propagating downward). + named params (propagating downward). ``unordered_when`` names a parent param + that must be exactly ``True`` to sort the projected child hashes. Duplicates + are retained; stored child order and the content hash are not changed. * ``Evaluate.Exclude()`` — drop the child entirely from the eval hash. * ``Evaluate.Unwrap()`` — mark a wrapper passthrough slot. When an identifier of the owning type is projected as a behavioral child, the eval hash "looks @@ -61,10 +63,14 @@ class Include(EvalMarker): child's subtree to these param names (overriding the child type's own projection and propagating downward). ``None`` means use the child type's own projection. + unordered_when (str | None): For a child field, the parent boolean param + that opts this list into order-independent evaluation identity. + Missing or non-true values keep the list ordered. """ fallback: str | None = None only_params: frozenset[str] | None = None + unordered_when: str | None = None @dataclass(frozen=True) diff --git a/pyrit/models/identifiers/scorer_identifier.py b/pyrit/models/identifiers/scorer_identifier.py index 08ce2b7858..37225d3da6 100644 --- a/pyrit/models/identifiers/scorer_identifier.py +++ b/pyrit/models/identifiers/scorer_identifier.py @@ -31,6 +31,10 @@ class ScorerIdentifier(ComponentIdentifier): ``chat_target`` constructor arg, and ``sub_scorers`` is an included parameter aliased to the composite scorer's ``scorers`` arg. Their identifier types make them references resolved by name from the target and scorer registries. + + The optional ``sub_scorers_order_independent`` param declares that the + aggregation verdict does not depend on child order. Legacy identifiers omit + it and remain ordered; aggregator names alone cannot prove commutativity. """ component_type: ClassVar[ComponentType] = ComponentType.SCORER @@ -44,6 +48,8 @@ class ScorerIdentifier(ComponentIdentifier): prompt_target: Annotated[TargetIdentifier | None, Evaluate.Include(), Param.Include(alias="chat_target")] = None #: Nested scorers a composite wraps, typed recursively. The composite #: constructor arg is ``scorers`` (a list), so the build marker aliases it. - sub_scorers: Annotated[list[ScorerIdentifier], Evaluate.Include(), Param.Include(alias="scorers")] = Field( - default_factory=list - ) + sub_scorers: Annotated[ + list[ScorerIdentifier], + Evaluate.Include(unordered_when="sub_scorers_order_independent"), + Param.Include(alias="scorers"), + ] = Field(default_factory=list) diff --git a/pyrit/score/true_false/true_false_composite_scorer.py b/pyrit/score/true_false/true_false_composite_scorer.py index 69e377c1e2..b42e3927f4 100644 --- a/pyrit/score/true_false/true_false_composite_scorer.py +++ b/pyrit/score/true_false/true_false_composite_scorer.py @@ -20,7 +20,7 @@ ScoringExpectation, ) from pyrit.score.observation.execution import _merge_observation_ids -from pyrit.score.true_false.true_false_score_aggregator import TrueFalseAggregatorFunc +from pyrit.score.true_false.true_false_score_aggregator import TrueFalseAggregatorFunc, TrueFalseScoreAggregator from pyrit.score.true_false.true_false_scorer import TrueFalseScorer logger = logging.getLogger(__name__) @@ -37,6 +37,12 @@ class TrueFalseCompositeScorer(TrueFalseScorer): Children are true/false scorers of any evidence kind, so a scorer over a message can be composed with one over evidence that is not a message at all. + + Built-in AND, OR, and MAJORITY aggregators opt into order-independent evaluation + identity. Child multiplicity still matters. Execution, rationale, and full content + identity retain child order. Custom aggregators remain ordered, even if their names + match a built-in. Legacy identifiers without the explicit opt-in stay ordered; + resuming those runs with a newly identified built-in composite requires a new run. """ def __init__( @@ -77,7 +83,16 @@ def _build_identifier(self) -> ComponentIdentifier: Returns: ComponentIdentifier: The identifier for this scorer. """ + order_independent = any( + self._score_aggregator is aggregator + for aggregator in ( + TrueFalseScoreAggregator.AND, + TrueFalseScoreAggregator.OR, + TrueFalseScoreAggregator.MAJORITY, + ) + ) return self._create_identifier( + params={"sub_scorers_order_independent": True} if order_independent else None, score_aggregator=self._score_aggregator.__name__, # type: ignore[ty:unresolved-attribute] sub_scorers=[s.get_identifier() for s in self._scorers], ) diff --git a/tests/unit/models/identifiers/test_evaluation_identifier.py b/tests/unit/models/identifiers/test_evaluation_identifier.py index 187cae6b2d..ce41bee188 100644 --- a/tests/unit/models/identifiers/test_evaluation_identifier.py +++ b/tests/unit/models/identifiers/test_evaluation_identifier.py @@ -13,6 +13,7 @@ import pytest from pyrit.models.identifiers import ( + AtomicAttackEvaluationIdentifier, ComponentIdentifier, compute_eval_hash, ) @@ -141,6 +142,61 @@ def test_returns_64_char_hex(self): assert all(c in "0123456789abcdef" for c in result) +class TestConditionalUnorderedChildren: + @pytest.mark.parametrize("opt_in", [None, False, True, "true", 1]) + def test_only_explicit_true_sorts_projected_hashes(self, opt_in: bool | str | int | None) -> None: + children = [ + ComponentIdentifier(class_name="Child", class_module="test", params={"value": value}) + for value in ("a", "b", "a") + ] + parent = ComponentIdentifier( + class_name="Parent", + class_module="test", + params={"unordered": opt_in}, + children={"items": children}, + ) + original = parent.model_dump_json() + result = _build_eval_dict(parent, child_eval_rules={"items": ChildEvalRule(unordered_when="unordered")}) + hashes = [child.hash for child in children] + + assert result["children"]["items"] == (sorted(hashes) if opt_in is True else hashes) + assert len(result["children"]["items"]) == 3 + assert parent.model_dump_json() == original + + def test_sort_uses_filtered_child_hashes_not_content_hashes(self) -> None: + children = [ + ComponentIdentifier(class_name="Target", class_module="test", params={"model": model, "endpoint": endpoint}) + for model, endpoint in [("a", "first"), ("b", "second")] + ] + parent = ComponentIdentifier( + class_name="Parent", class_module="test", params={"unordered": True}, children={"items": children} + ) + rule = ChildEvalRule(unordered_when="unordered", included_params=frozenset({"model"})) + result = _build_eval_dict(parent, child_eval_rules={"items": rule}) + expected = [ + ComponentIdentifier(class_name="Target", class_module="test", params={"model": model}).hash + for model in ("a", "b") + ] + assert result["children"]["items"] == sorted(expected) + + def test_order_sensitive_converter_slots_remain_ordered(self) -> None: + converters = [ + ComponentIdentifier(class_name=name, class_module="pyrit.converter") for name in ("First", "Second") + ] + first = ComponentIdentifier( + class_name="Attack", class_module="test", children={"request_converters": converters} + ) + second = ComponentIdentifier( + class_name="Attack", class_module="test", children={"request_converters": list(reversed(converters))} + ) + assert AtomicAttackEvaluationIdentifier(first).eval_hash != AtomicAttackEvaluationIdentifier(second).eval_hash + + def test_unordered_rule_is_invalid_for_root_params(self) -> None: + parent = ComponentIdentifier(class_name="Parent", class_module="test") + with pytest.raises(ValueError, match="own_rule.unordered_when"): + compute_eval_hash(parent, child_eval_rules={}, own_rule=ChildEvalRule(unordered_when="unordered")) + + class TestEvaluationIdentifier: """Tests for the EvaluationIdentifier abstract base class.""" diff --git a/tests/unit/models/identifiers/test_evaluation_markers.py b/tests/unit/models/identifiers/test_evaluation_markers.py index b512721a6c..af697c062b 100644 --- a/tests/unit/models/identifiers/test_evaluation_markers.py +++ b/tests/unit/models/identifiers/test_evaluation_markers.py @@ -37,10 +37,12 @@ def test_include_defaults_and_fields(self): plain = Include() assert plain.fallback is None assert plain.only_params is None + assert plain.unordered_when is None configured = Include(fallback="model_name", only_params=frozenset({"temperature"})) assert configured.fallback == "model_name" assert configured.only_params == frozenset({"temperature"}) + assert Include(unordered_when="commutative").unordered_when == "commutative" def test_markers_are_frozen(self): marker = Include() @@ -79,6 +81,11 @@ def test_attack_objective_target_only_params(self): assert isinstance(self._marker(AttackIdentifier, "objective_scorer"), Exclude) + def test_scorer_child_order_marker(self) -> None: + marker = self._marker(ScorerIdentifier, "sub_scorers") + assert isinstance(marker, Include) + assert marker.unordered_when == "sub_scorers_order_independent" + def _field_marker(model_cls, field_name): """Return the ``EvalMarker`` attached to a field, or ``None``.""" diff --git a/tests/unit/scenario/core/test_scenario.py b/tests/unit/scenario/core/test_scenario.py index 8d6fd09973..8461b7573d 100644 --- a/tests/unit/scenario/core/test_scenario.py +++ b/tests/unit/scenario/core/test_scenario.py @@ -31,7 +31,8 @@ from pyrit.scenario.core import AtomicAttack, BaselineAttackPolicy, Scenario, ScenarioTechnique from pyrit.scenario.core.matrix_atomic_attack_builder import build_baseline_atomic_attack from pyrit.scenario.core.scenario_context import ScenarioContext -from pyrit.score import Scorer +from pyrit.score import Scorer, SubStringScorer, TrueFalseCompositeScorer, TrueFalseScoreAggregator +from pyrit.score.true_false.true_false_score_aggregator import TrueFalseAggregatorFunc from tests.unit.mocks import make_scenario_identifier, make_scenario_result # Reusable test scorer identifier @@ -1554,6 +1555,51 @@ def test_raises_when_version_mismatches(self): class TestScenarioResumption: """Tests for scenario resumption logic in initialize_async.""" + @pytest.mark.parametrize("aggregator", [TrueFalseScoreAggregator.OR, TrueFalseScoreAggregator.AND]) + @pytest.mark.parametrize("replacement_substrings", [["b", "a"], ["a", "c"], ["a", "b", "b"]]) + async def test_resume_with_composite_scorer_async( + self, + mock_objective_target: PromptTarget, + aggregator: TrueFalseAggregatorFunc, + replacement_substrings: list[str], + ) -> None: + scorer = TrueFalseCompositeScorer( + aggregator=aggregator, scorers=[SubStringScorer(substring=value) for value in ("a", "b")] + ) + dataset_config = MagicMock(spec=DatasetAttackConfiguration) + dataset_config.get_attack_groups_by_dataset_async.return_value = { + "default": [AttackSeedGroup(seeds=[SeedObjective(value="test objective")])] + } + args = {"objective_target": mock_objective_target, "dataset_config": dataset_config} + original = ConcreteScenarioWithTrueFalseScorer(name="Composite resume", version=1, objective_scorer=scorer) + original.set_params_from_args(args=args) + await original.initialize_async() + assert original.atomic_attack_count == 1 + original_id = original._scenario_result_id + header = original._memory.get_scenario_result_header(scenario_result_id=original_id) + assert header is not None + stored_plan = header.metadata[SCENARIO_RUN_PLAN_METADATA_KEY] + + replacement = TrueFalseCompositeScorer( + aggregator=aggregator, + scorers=[SubStringScorer(substring=value) for value in replacement_substrings], + ) + resumed = ConcreteScenarioWithTrueFalseScorer( + name="Composite resume", version=1, objective_scorer=replacement, scenario_result_id=original_id + ) + resumed.set_params_from_args(args=args) + if replacement_substrings != ["b", "a"]: + with pytest.raises(ValueError, match="does not match the current"): + await resumed.initialize_async() + return + + await resumed.initialize_async() + assert resumed._scenario_result_id == original_id + assert resumed._atomic_attacks[0].objectives == ["test objective"] + resumed_header = resumed._memory.get_scenario_result_header(scenario_result_id=original_id) + assert resumed_header is not None + assert resumed_header.metadata[SCENARIO_RUN_PLAN_METADATA_KEY] == stored_plan + async def test_resume_succeeds_when_stored_result_matches(self, mock_objective_target, mock_atomic_attacks): """When scenario_result_id finds a matching result, no new result is created.""" scenario = ConcreteScenario( diff --git a/tests/unit/score/test_scorer_evaluation_identifier.py b/tests/unit/score/test_scorer_evaluation_identifier.py index 937ada64c2..eb094f2d77 100644 --- a/tests/unit/score/test_scorer_evaluation_identifier.py +++ b/tests/unit/score/test_scorer_evaluation_identifier.py @@ -7,9 +7,15 @@ Covers ``ScorerEvaluationIdentifier`` ClassVar values and eval-hash delegation. """ +from collections.abc import Iterable +from itertools import permutations + import pytest -from pyrit.models import ComponentIdentifier, Identifiable, ScorerEvaluationIdentifier, compute_eval_hash +from pyrit.models import ComponentIdentifier, Identifiable, Score, ScorerEvaluationIdentifier, compute_eval_hash +from pyrit.score import SubStringScorer, TrueFalseCompositeScorer, TrueFalseScoreAggregator +from pyrit.score.score_aggregator_result import ScoreAggregatorResult +from pyrit.score.true_false.true_false_score_aggregator import TrueFalseAggregatorFunc class TestScorerEvaluationIdentifierConstants: @@ -22,7 +28,11 @@ def test_child_eval_rules_keys(self): is the global wrapper-passthrough rule derived from ``TargetIdentifier.targets`` (it only fires on nested multi-targets). """ - assert set(ScorerEvaluationIdentifier.CHILD_EVAL_RULES.keys()) == {"prompt_target", "targets"} + assert set(ScorerEvaluationIdentifier.CHILD_EVAL_RULES.keys()) == {"prompt_target", "targets", "sub_scorers"} + + def test_sub_scorers_require_explicit_order_independence(self) -> None: + rule = ScorerEvaluationIdentifier.CHILD_EVAL_RULES["sub_scorers"] + assert rule.unordered_when == "sub_scorers_order_independent" def test_prompt_target_rule(self): """Test that prompt_target has the expected included params and fallbacks.""" @@ -87,6 +97,99 @@ def test_eval_hash_matches_free_function(self): assert identity.eval_hash == expected +@pytest.mark.usefixtures("patch_central_database") +class TestCompositeEvaluationOrder: + @pytest.mark.parametrize( + "aggregator", + [TrueFalseScoreAggregator.OR, TrueFalseScoreAggregator.AND, TrueFalseScoreAggregator.MAJORITY], + ) + def test_permutations_preserve_eval_identity_and_content_order(self, aggregator: TrueFalseAggregatorFunc) -> None: + children = [SubStringScorer(substring=value) for value in ("a", "b", "c")] + identifiers = [ + TrueFalseCompositeScorer(aggregator=aggregator, scorers=list(order)).get_identifier() + for order in permutations(children) + ] + + assert len({ScorerEvaluationIdentifier(identifier).eval_hash for identifier in identifiers}) == 1 + assert len({identifier.hash for identifier in identifiers}) == 6 + for order, identifier in zip(permutations(children), identifiers, strict=True): + assert [child.hash for child in identifier.get_child_list("sub_scorers")] == [ + child.get_identifier().hash for child in order + ] + restored = ComponentIdentifier.model_validate_json(identifier.model_dump_json()) + assert restored.hash == identifier.hash + assert ScorerEvaluationIdentifier(restored).eval_hash == identifier.eval_hash + + def test_nested_permutations_preserve_eval_identity(self) -> None: + a, b, c = [SubStringScorer(substring=value) for value in ("a", "b", "c")] + first = TrueFalseCompositeScorer( + aggregator=TrueFalseScoreAggregator.OR, + scorers=[TrueFalseCompositeScorer(aggregator=TrueFalseScoreAggregator.AND, scorers=[a, b]), c], + ) + second = TrueFalseCompositeScorer( + aggregator=TrueFalseScoreAggregator.OR, + scorers=[c, TrueFalseCompositeScorer(aggregator=TrueFalseScoreAggregator.AND, scorers=[b, a])], + ) + assert first.get_identifier().eval_hash == second.get_identifier().eval_hash + assert first.get_identifier().hash != second.get_identifier().hash + + def test_configuration_aggregator_and_multiplicity_remain_distinct(self) -> None: + a, b, c = [SubStringScorer(substring=value) for value in ("a", "b", "c")] + configurations = [ + (TrueFalseScoreAggregator.OR, [a, b]), + (TrueFalseScoreAggregator.OR, [a, c]), + (TrueFalseScoreAggregator.AND, [a, b]), + (TrueFalseScoreAggregator.MAJORITY, [a, b]), + (TrueFalseScoreAggregator.MAJORITY, [a, a, b]), + (TrueFalseScoreAggregator.MAJORITY, [a, b, b]), + ] + hashes = { + TrueFalseCompositeScorer(aggregator=aggregator, scorers=scorers).get_identifier().eval_hash + for aggregator, scorers in configurations + } + assert len(hashes) == len(configurations) + + def test_custom_aggregator_with_builtin_name_remains_ordered(self) -> None: + def first_score(scores: Iterable[Score]) -> ScoreAggregatorResult: + return TrueFalseScoreAggregator.OR([next(iter(scores))]) + + first_score.__name__ = TrueFalseScoreAggregator.OR.__name__ + a, b = [SubStringScorer(substring=value) for value in ("a", "b")] + first = TrueFalseCompositeScorer(aggregator=first_score, scorers=[a, b]).get_identifier() + second = TrueFalseCompositeScorer(aggregator=first_score, scorers=[b, a]).get_identifier() + builtin = TrueFalseCompositeScorer(aggregator=TrueFalseScoreAggregator.OR, scorers=[a, b]).get_identifier() + + assert "sub_scorers_order_independent" not in first.params + assert first.eval_hash != second.eval_hash + assert first.eval_hash != builtin.eval_hash + outer_first = TrueFalseCompositeScorer( + aggregator=TrueFalseScoreAggregator.AND, + scorers=[TrueFalseCompositeScorer(aggregator=first_score, scorers=[a, b]), a], + ) + outer_second = TrueFalseCompositeScorer( + aggregator=TrueFalseScoreAggregator.AND, + scorers=[a, TrueFalseCompositeScorer(aggregator=first_score, scorers=[b, a])], + ) + assert outer_first.get_identifier().eval_hash != outer_second.get_identifier().eval_hash + + def test_legacy_identifiers_are_not_inferred_from_aggregator_names(self) -> None: + a, b = [SubStringScorer(substring=value) for value in ("a", "b")] + current = TrueFalseCompositeScorer(aggregator=TrueFalseScoreAggregator.OR, scorers=[a, b]).get_identifier() + legacy_data = current.model_dump() + del legacy_data["sub_scorers_order_independent"] + legacy = ComponentIdentifier.model_validate(legacy_data) + reversed_legacy = ComponentIdentifier( + class_name=legacy.class_name, + class_module=legacy.class_module, + params=legacy.params, + children={"sub_scorers": list(reversed(legacy.get_child_list("sub_scorers")))}, + ) + + assert ScorerEvaluationIdentifier(legacy).eval_hash != ScorerEvaluationIdentifier(reversed_legacy).eval_hash + assert ScorerEvaluationIdentifier(legacy).eval_hash != current.eval_hash + assert legacy.hash != current.hash + + @pytest.mark.usefixtures("patch_central_database") class TestScorerGetEvalHash: """Tests for ScorerEvaluationIdentifier eval_hash computation.""" From 96a884e6efb10166020805567bda3b958b59153e Mon Sep 17 00:00:00 2001 From: Richard Lundeen Date: Tue, 22 Sep 2026 17:17:40 -0700 Subject: [PATCH 2/2] DOC: Keep scorer identity documentation concise Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- doc/code/memory/3_memory_data_types.md | 22 ------------------- pyrit/models/identifiers/scorer_identifier.py | 3 +-- .../true_false/true_false_composite_scorer.py | 5 +---- 3 files changed, 2 insertions(+), 28 deletions(-) diff --git a/doc/code/memory/3_memory_data_types.md b/doc/code/memory/3_memory_data_types.md index 4df2604524..7ab05d53bf 100644 --- a/doc/code/memory/3_memory_data_types.md +++ b/doc/code/memory/3_memory_data_types.md @@ -165,25 +165,3 @@ What feeds the eval hash is declared **on the strongly-typed identifier fields t - `Evaluate.Unwrap()` — mark a wrapper passthrough slot (e.g. `TargetIdentifier.targets`). A multi-target like `RoundRobinTarget` is "looked through" to its inner target, so it eval-hashes the same as the bare inner target. For example, `TargetIdentifier` excludes `endpoint` but includes `temperature`, and the `ObjectiveTargetEvaluationIdentifier` / `ScorerEvaluationIdentifier` / `AtomicAttackEvaluationIdentifier` subclasses derive their engine rules from these markers (via `derive_eval_config`). Markers affect **only** the eval hash — the identity `hash` always keeps distinct components (e.g. a wrapper vs. its inner target) distinct. - -#### Child order - -Child lists are ordered unless a child field declares -`Evaluate.Include(unordered_when="parent_param")` and that parent param is exactly -`True`. The eval engine then sorts the child hashes **after** behavioral filtering. -It retains duplicates and does not change the stored list or execution order. - -`TrueFalseCompositeScorer` sets the `sub_scorers_order_independent` param only for -the built-in `AND`, `OR`, and `MAJORITY` aggregator functions. Reordered children -then have the same eval hash, including within nested composites and scenario -resume checks. Different child configurations and duplicate counts remain distinct. -Custom aggregators, converter pipelines, and other sequences keep their order. -Rationale and metadata output can still follow input order. - -**Compatibility:** The new opt-in param changes the content and eval hashes of -built-in composites and identifiers that include them. Existing records without -the param remain ordered: an aggregator name alone cannot prove that a custom -function is commutative. No stored hashes are migrated. A scenario saved before -this opt-in cannot resume with a new built-in composite; start a new run. -Stored eval hashes are recomputed when wrapped in an `EvaluationIdentifier`, -but hash-only lookup keys and cached evaluation results are not remapped. diff --git a/pyrit/models/identifiers/scorer_identifier.py b/pyrit/models/identifiers/scorer_identifier.py index 37225d3da6..3d007e29e8 100644 --- a/pyrit/models/identifiers/scorer_identifier.py +++ b/pyrit/models/identifiers/scorer_identifier.py @@ -33,8 +33,7 @@ class ScorerIdentifier(ComponentIdentifier): them references resolved by name from the target and scorer registries. The optional ``sub_scorers_order_independent`` param declares that the - aggregation verdict does not depend on child order. Legacy identifiers omit - it and remain ordered; aggregator names alone cannot prove commutativity. + aggregation verdict does not depend on child order. """ component_type: ClassVar[ComponentType] = ComponentType.SCORER diff --git a/pyrit/score/true_false/true_false_composite_scorer.py b/pyrit/score/true_false/true_false_composite_scorer.py index b42e3927f4..7e31911a2b 100644 --- a/pyrit/score/true_false/true_false_composite_scorer.py +++ b/pyrit/score/true_false/true_false_composite_scorer.py @@ -39,10 +39,7 @@ class TrueFalseCompositeScorer(TrueFalseScorer): composed with one over evidence that is not a message at all. Built-in AND, OR, and MAJORITY aggregators opt into order-independent evaluation - identity. Child multiplicity still matters. Execution, rationale, and full content - identity retain child order. Custom aggregators remain ordered, even if their names - match a built-in. Legacy identifiers without the explicit opt-in stay ordered; - resuming those runs with a newly identified built-in composite requires a new run. + identity. Duplicates remain significant; custom aggregators and execution stay ordered. """ def __init__(