[BREAKING] FIX: Make commutative scorer evaluation identity order-independent

pyrit/models/identifiers/evaluation_identifier.py

@@ -82,6 +82,9 @@ class ChildEvalRule(BaseModel):
       ``RoundRobinTarget``). The first item of that sub-child list is
       substituted before applying param filtering, so the eval hash
       matches the unwrapped inner target. ``None`` means no unwrapping.
+    * ``unordered_when`` — names a boolean param on the parent identifier.
+      Only an explicit ``True`` sorts this slot's projected child hashes.
+      Duplicates are retained, and the original child list is not changed.
     """
 
     model_config = ConfigDict(frozen=True)
@@ -91,6 +94,7 @@ class ChildEvalRule(BaseModel):
     included_item_values: dict[str, Any] | None = Field(default=None)
     param_fallbacks: dict[str, str] | None = Field(default=None)
     inner_child_name: str | None = Field(default=None)
+    unordered_when: str | None = None
 
 
 def _build_eval_dict(
@@ -188,6 +192,8 @@ def _build_eval_dict(
                 )
                 for c in child_list
             ]
+            if rule and rule.unordered_when and identifier.params.get(rule.unordered_when) is True:
+                hashes.sort()
             eval_children[name] = hashes[0] if len(hashes) == 1 else hashes
         if eval_children:
             eval_dict["children"] = eval_children
@@ -227,7 +233,7 @@ def compute_eval_hash(
         own_rule (ChildEvalRule | None): Rule applied to the root entity's
             own params and fallbacks. Only ``included_params`` and
             ``param_fallbacks`` are honored; ``exclude``, ``included_item_values``,
-            and ``inner_child_name`` are not meaningful at the root and will
+            ``inner_child_name``, and ``unordered_when`` are not meaningful at the root and will
             raise ``ValueError`` if set. Defaults to None.
         root_unwrap_child (str | None): If set, names the wrapper passthrough
             slot on the root identifier (e.g. ``"targets"``). When the root is a
@@ -249,6 +255,8 @@ def compute_eval_hash(
             raise ValueError("own_rule.included_item_values is not meaningful at the root entity")
         if own_rule.inner_child_name is not None:
             raise ValueError("own_rule.inner_child_name is not meaningful at the root entity")
+        if own_rule.unordered_when is not None:
+            raise ValueError("own_rule.unordered_when is not meaningful at the root entity")
 
     if root_unwrap_child is not None:
         inner = identifier.get_child_list(root_unwrap_child)
@@ -310,6 +318,7 @@ def _slot_rule(
         included_params=included_params,
         param_fallbacks=fallbacks,
         inner_child_name=_type_unwrap_field(child_type),
+        unordered_when=marker.unordered_when if isinstance(marker, Include) else None,
     )
 
 
@@ -321,6 +330,7 @@ def _is_neutral_rule(rule: ChildEvalRule) -> bool:
         and rule.included_item_values is None
         and rule.param_fallbacks is None
         and rule.inner_child_name is None
+        and rule.unordered_when is None
     )
 
 

pyrit/models/identifiers/evaluation_markers.py

@@ -29,7 +29,9 @@ class TargetIdentifier(ComponentIdentifier):
   * ``Evaluate.Include()`` — include the child, projecting its subtree with the
     child type's own markers (the default for an unmarked field). ``only_params``
     overrides that projection for this slot, restricting the child subtree to the
-    named params (propagating downward).
+    named params (propagating downward). ``unordered_when`` names a parent param
+    that must be exactly ``True`` to sort the projected child hashes. Duplicates
+    are retained; stored child order and the content hash are not changed.
   * ``Evaluate.Exclude()`` — drop the child entirely from the eval hash.
   * ``Evaluate.Unwrap()`` — mark a wrapper passthrough slot. When an identifier of
     the owning type is projected as a behavioral child, the eval hash "looks
@@ -61,10 +63,14 @@ class Include(EvalMarker):
             child's subtree to these param names (overriding the child type's own
             projection and propagating downward). ``None`` means use the child
             type's own projection.
+        unordered_when (str | None): For a child field, the parent boolean param
+            that opts this list into order-independent evaluation identity.
+            Missing or non-true values keep the list ordered.
     """
 
     fallback: str | None = None
     only_params: frozenset[str] | None = None
+    unordered_when: str | None = None
 
 
 @dataclass(frozen=True)

pyrit/models/identifiers/scorer_identifier.py

@@ -31,6 +31,9 @@ class ScorerIdentifier(ComponentIdentifier):
     ``chat_target`` constructor arg, and ``sub_scorers`` is an included parameter
     aliased to the composite scorer's ``scorers`` arg. Their identifier types make
     them references resolved by name from the target and scorer registries.
+
+    The optional ``sub_scorers_order_independent`` param declares that the
+    aggregation verdict does not depend on child order.
     """
 
     component_type: ClassVar[ComponentType] = ComponentType.SCORER
@@ -44,6 +47,8 @@ class ScorerIdentifier(ComponentIdentifier):
     prompt_target: Annotated[TargetIdentifier | None, Evaluate.Include(), Param.Include(alias="chat_target")] = None
     #: Nested scorers a composite wraps, typed recursively. The composite
     #: constructor arg is ``scorers`` (a list), so the build marker aliases it.
-    sub_scorers: Annotated[list[ScorerIdentifier], Evaluate.Include(), Param.Include(alias="scorers")] = Field(
-        default_factory=list
-    )
+    sub_scorers: Annotated[
+        list[ScorerIdentifier],
+        Evaluate.Include(unordered_when="sub_scorers_order_independent"),
+        Param.Include(alias="scorers"),
+    ] = Field(default_factory=list)

pyrit/score/true_false/true_false_composite_scorer.py

@@ -20,7 +20,7 @@
     ScoringExpectation,
 )
 from pyrit.score.observation.execution import _merge_observation_ids
-from pyrit.score.true_false.true_false_score_aggregator import TrueFalseAggregatorFunc
+from pyrit.score.true_false.true_false_score_aggregator import TrueFalseAggregatorFunc, TrueFalseScoreAggregator
 from pyrit.score.true_false.true_false_scorer import TrueFalseScorer
 
 logger = logging.getLogger(__name__)
@@ -37,6 +37,9 @@ class TrueFalseCompositeScorer(TrueFalseScorer):
 
     Children are true/false scorers of any evidence kind, so a scorer over a message can be
     composed with one over evidence that is not a message at all.
+
+    Built-in AND, OR, and MAJORITY aggregators opt into order-independent evaluation
+    identity. Duplicates remain significant; custom aggregators and execution stay ordered.
     """
 
     def __init__(
@@ -77,7 +80,16 @@ def _build_identifier(self) -> ComponentIdentifier:
         Returns:
             ComponentIdentifier: The identifier for this scorer.
         """
+        order_independent = any(
+            self._score_aggregator is aggregator
+            for aggregator in (
+                TrueFalseScoreAggregator.AND,
+                TrueFalseScoreAggregator.OR,
+                TrueFalseScoreAggregator.MAJORITY,
+            )
+        )
         return self._create_identifier(
+            params={"sub_scorers_order_independent": True} if order_independent else None,
             score_aggregator=self._score_aggregator.__name__,  # type: ignore[ty:unresolved-attribute]
             sub_scorers=[s.get_identifier() for s in self._scorers],
         )

tests/unit/models/identifiers/test_evaluation_identifier.py

@@ -13,6 +13,7 @@
 import pytest
 
 from pyrit.models.identifiers import (
+    AtomicAttackEvaluationIdentifier,
     ComponentIdentifier,
     compute_eval_hash,
 )
@@ -141,6 +142,61 @@ def test_returns_64_char_hex(self):
         assert all(c in "0123456789abcdef" for c in result)
 
 
+class TestConditionalUnorderedChildren:
+    @pytest.mark.parametrize("opt_in", [None, False, True, "true", 1])
+    def test_only_explicit_true_sorts_projected_hashes(self, opt_in: bool | str | int | None) -> None:
+        children = [
+            ComponentIdentifier(class_name="Child", class_module="test", params={"value": value})
+            for value in ("a", "b", "a")
+        ]
+        parent = ComponentIdentifier(
+            class_name="Parent",
+            class_module="test",
+            params={"unordered": opt_in},
+            children={"items": children},
+        )
+        original = parent.model_dump_json()
+        result = _build_eval_dict(parent, child_eval_rules={"items": ChildEvalRule(unordered_when="unordered")})
+        hashes = [child.hash for child in children]
+
+        assert result["children"]["items"] == (sorted(hashes) if opt_in is True else hashes)
+        assert len(result["children"]["items"]) == 3
+        assert parent.model_dump_json() == original
+
+    def test_sort_uses_filtered_child_hashes_not_content_hashes(self) -> None:
+        children = [
+            ComponentIdentifier(class_name="Target", class_module="test", params={"model": model, "endpoint": endpoint})
+            for model, endpoint in [("a", "first"), ("b", "second")]
+        ]
+        parent = ComponentIdentifier(
+            class_name="Parent", class_module="test", params={"unordered": True}, children={"items": children}
+        )
+        rule = ChildEvalRule(unordered_when="unordered", included_params=frozenset({"model"}))
+        result = _build_eval_dict(parent, child_eval_rules={"items": rule})
+        expected = [
+            ComponentIdentifier(class_name="Target", class_module="test", params={"model": model}).hash
+            for model in ("a", "b")
+        ]
+        assert result["children"]["items"] == sorted(expected)
+
+    def test_order_sensitive_converter_slots_remain_ordered(self) -> None:
+        converters = [
+            ComponentIdentifier(class_name=name, class_module="pyrit.converter") for name in ("First", "Second")
+        ]
+        first = ComponentIdentifier(
+            class_name="Attack", class_module="test", children={"request_converters": converters}
+        )
+        second = ComponentIdentifier(
+            class_name="Attack", class_module="test", children={"request_converters": list(reversed(converters))}
+        )
+        assert AtomicAttackEvaluationIdentifier(first).eval_hash != AtomicAttackEvaluationIdentifier(second).eval_hash
+
+    def test_unordered_rule_is_invalid_for_root_params(self) -> None:
+        parent = ComponentIdentifier(class_name="Parent", class_module="test")
+        with pytest.raises(ValueError, match="own_rule.unordered_when"):
+            compute_eval_hash(parent, child_eval_rules={}, own_rule=ChildEvalRule(unordered_when="unordered"))
+
+
 class TestEvaluationIdentifier:
     """Tests for the EvaluationIdentifier abstract base class."""
 

tests/unit/models/identifiers/test_evaluation_markers.py

@@ -37,10 +37,12 @@ def test_include_defaults_and_fields(self):
         plain = Include()
         assert plain.fallback is None
         assert plain.only_params is None
+        assert plain.unordered_when is None
 
         configured = Include(fallback="model_name", only_params=frozenset({"temperature"}))
         assert configured.fallback == "model_name"
         assert configured.only_params == frozenset({"temperature"})
+        assert Include(unordered_when="commutative").unordered_when == "commutative"
 
     def test_markers_are_frozen(self):
         marker = Include()
@@ -79,6 +81,11 @@ def test_attack_objective_target_only_params(self):
 
         assert isinstance(self._marker(AttackIdentifier, "objective_scorer"), Exclude)
 
+    def test_scorer_child_order_marker(self) -> None:
+        marker = self._marker(ScorerIdentifier, "sub_scorers")
+        assert isinstance(marker, Include)
+        assert marker.unordered_when == "sub_scorers_order_independent"
+
 
 def _field_marker(model_cls, field_name):
     """Return the ``EvalMarker`` attached to a field, or ``None``."""

tests/unit/scenario/core/test_scenario.py

@@ -31,7 +31,8 @@
 from pyrit.scenario.core import AtomicAttack, BaselineAttackPolicy, Scenario, ScenarioTechnique
 from pyrit.scenario.core.matrix_atomic_attack_builder import build_baseline_atomic_attack
 from pyrit.scenario.core.scenario_context import ScenarioContext
-from pyrit.score import Scorer
+from pyrit.score import Scorer, SubStringScorer, TrueFalseCompositeScorer, TrueFalseScoreAggregator
+from pyrit.score.true_false.true_false_score_aggregator import TrueFalseAggregatorFunc
 from tests.unit.mocks import make_scenario_identifier, make_scenario_result
 
 # Reusable test scorer identifier
@@ -1554,6 +1555,51 @@ def test_raises_when_version_mismatches(self):
 class TestScenarioResumption:
     """Tests for scenario resumption logic in initialize_async."""
 
+    @pytest.mark.parametrize("aggregator", [TrueFalseScoreAggregator.OR, TrueFalseScoreAggregator.AND])
+    @pytest.mark.parametrize("replacement_substrings", [["b", "a"], ["a", "c"], ["a", "b", "b"]])
+    async def test_resume_with_composite_scorer_async(
+        self,
+        mock_objective_target: PromptTarget,
+        aggregator: TrueFalseAggregatorFunc,
+        replacement_substrings: list[str],
+    ) -> None:
+        scorer = TrueFalseCompositeScorer(
+            aggregator=aggregator, scorers=[SubStringScorer(substring=value) for value in ("a", "b")]
+        )
+        dataset_config = MagicMock(spec=DatasetAttackConfiguration)
+        dataset_config.get_attack_groups_by_dataset_async.return_value = {
+            "default": [AttackSeedGroup(seeds=[SeedObjective(value="test objective")])]
+        }
+        args = {"objective_target": mock_objective_target, "dataset_config": dataset_config}
+        original = ConcreteScenarioWithTrueFalseScorer(name="Composite resume", version=1, objective_scorer=scorer)
+        original.set_params_from_args(args=args)
+        await original.initialize_async()
+        assert original.atomic_attack_count == 1
+        original_id = original._scenario_result_id
+        header = original._memory.get_scenario_result_header(scenario_result_id=original_id)
+        assert header is not None
+        stored_plan = header.metadata[SCENARIO_RUN_PLAN_METADATA_KEY]
+
+        replacement = TrueFalseCompositeScorer(
+            aggregator=aggregator,
+            scorers=[SubStringScorer(substring=value) for value in replacement_substrings],
+        )
+        resumed = ConcreteScenarioWithTrueFalseScorer(
+            name="Composite resume", version=1, objective_scorer=replacement, scenario_result_id=original_id
+        )
+        resumed.set_params_from_args(args=args)
+        if replacement_substrings != ["b", "a"]:
+            with pytest.raises(ValueError, match="does not match the current"):
+                await resumed.initialize_async()
+            return
+
+        await resumed.initialize_async()
+        assert resumed._scenario_result_id == original_id
+        assert resumed._atomic_attacks[0].objectives == ["test objective"]
+        resumed_header = resumed._memory.get_scenario_result_header(scenario_result_id=original_id)
+        assert resumed_header is not None
+        assert resumed_header.metadata[SCENARIO_RUN_PLAN_METADATA_KEY] == stored_plan
+
     async def test_resume_succeeds_when_stored_result_matches(self, mock_objective_target, mock_atomic_attacks):
         """When scenario_result_id finds a matching result, no new result is created."""
         scenario = ConcreteScenario(