diff --git a/pyrit/exceptions/__init__.py b/pyrit/exceptions/__init__.py index 5b4948a628..6d889ab95d 100644 --- a/pyrit/exceptions/__init__.py +++ b/pyrit/exceptions/__init__.py @@ -12,6 +12,7 @@ MissingPromptPlaceholderException, PyritException, RateLimitException, + ScorerLLMResponseBlockedException, get_retry_max_num_attempts, handle_bad_request_exception, pyrit_custom_result_retry, @@ -60,6 +61,7 @@ "RateLimitException", "remove_markdown_json", "RetryCollector", + "ScorerLLMResponseBlockedException", "set_execution_context", "set_retry_collector", "execution_context", diff --git a/pyrit/exceptions/exception_classes.py b/pyrit/exceptions/exception_classes.py index 5b4f3de72f..04043a2381 100644 --- a/pyrit/exceptions/exception_classes.py +++ b/pyrit/exceptions/exception_classes.py @@ -205,6 +205,21 @@ def __init__(self, *, status_code: int = 204, message: str = "No Content") -> No super().__init__(status_code=status_code, message=message) +class ScorerLLMResponseBlockedException(BadRequestException): + """Exception raised when a scorer's own LLM response is blocked by content filtering.""" + + def __init__(self, *, status_code: int = 400, message: str = "Scorer LLM response blocked") -> None: + """ + Initialize a scorer-response-blocked exception. + + Args: + status_code (int): Status code for the error. + message (str): Error message. + + """ + super().__init__(status_code=status_code, message=message) + + class InvalidJsonException(PyritException): """Exception class for blocked content errors.""" diff --git a/pyrit/score/float_scale/azure_content_filter_scorer.py b/pyrit/score/float_scale/azure_content_filter_scorer.py index 89cd2597cc..b4f4bfd84a 100644 --- a/pyrit/score/float_scale/azure_content_filter_scorer.py +++ b/pyrit/score/float_scale/azure_content_filter_scorer.py @@ -340,7 +340,9 @@ async def _score_piece_async(self, message_piece: MessagePiece, *, objective: st for result in aggregated_results ] - def _build_fallback_score(self, *, message: Message, objective: str | None) -> list[Score]: + def _build_fallback_score( + self, *, message: Message, objective: str | None, scorer_response_blocked: bool = False + ) -> list[Score]: """ Build one neutral ``0.0`` fallback score per configured harm category. @@ -357,6 +359,8 @@ def _build_fallback_score(self, *, message: Message, objective: str | None) -> l Args: message (Message): The message whose first piece is inspected for status. objective (str | None): The objective associated with this scoring call. + scorer_response_blocked (bool): When True, the scorer's own LLM response was + blocked by content filtering; reflected in the rationale. Returns: list[Score]: One ``0.0`` ``float_scale`` score per configured harm category, @@ -370,7 +374,10 @@ def _build_fallback_score(self, *, message: Message, objective: str | None) -> l if piece_id is None: raise ValueError("Cannot create score: message piece has no id or original_prompt_id") - if first_piece.is_blocked(): + if scorer_response_blocked: + status = "The scorer's own LLM response was blocked by content filtering (raise_if_scorer_blocks is False)" + description = "Scorer response blocked; returning 0.0 per configured category." + elif first_piece.is_blocked(): status = ( "The request was blocked by the target (score_blocked_content is False or no partial content available)" ) diff --git a/pyrit/score/float_scale/float_scale_scorer.py b/pyrit/score/float_scale/float_scale_scorer.py index 6f304c2e8f..538ea44d1c 100644 --- a/pyrit/score/float_scale/float_scale_scorer.py +++ b/pyrit/score/float_scale/float_scale_scorer.py @@ -49,7 +49,9 @@ def __init__(self, *, validator: ScorerPromptValidator, chat_target: PromptTarge """ super().__init__(validator=validator, chat_target=chat_target) - def _build_fallback_score(self, *, message: Message, objective: str | None) -> list[Score]: + def _build_fallback_score( + self, *, message: Message, objective: str | None, scorer_response_blocked: bool = False + ) -> list[Score]: """ Build a single-element list containing a neutral ``0.0`` score when no pieces could be scored. @@ -59,6 +61,8 @@ def _build_fallback_score(self, *, message: Message, objective: str | None) -> l Args: message (Message): The message whose first piece is inspected for status. objective (str | None): The objective associated with this scoring call. + scorer_response_blocked (bool): When True, the scorer's own LLM response was + blocked by content filtering; reflected in the rationale. Returns: list[Score]: A single-element list containing a ``0.0`` ``float_scale`` score @@ -72,7 +76,13 @@ def _build_fallback_score(self, *, message: Message, objective: str | None) -> l if piece_id is None: raise ValueError("Cannot create score: message piece has no id or original_prompt_id") - if first_piece.is_blocked(): + if scorer_response_blocked: + rationale = ( + "The scorer's own LLM response was blocked by content filtering " + "(raise_if_scorer_blocks is False); returning 0.0." + ) + description = "Scorer response blocked; returning 0.0." + elif first_piece.is_blocked(): rationale = ( "The request was blocked by the target " "(score_blocked_content is False or no partial content available); returning 0.0." diff --git a/pyrit/score/llm_scoring.py b/pyrit/score/llm_scoring.py index 9cf91e300d..78183f0a74 100644 --- a/pyrit/score/llm_scoring.py +++ b/pyrit/score/llm_scoring.py @@ -6,7 +6,7 @@ import uuid from typing import TYPE_CHECKING, Any -from pyrit.exceptions import pyrit_json_retry +from pyrit.exceptions import EmptyResponseException, ScorerLLMResponseBlockedException, pyrit_json_retry from pyrit.models import JSON_SCHEMA_METADATA_KEY, Message, MessagePiece if TYPE_CHECKING: @@ -75,6 +75,11 @@ async def _run_llm_scoring_async( normalized and validated by the caller. Raises: + ScorerLLMResponseBlockedException: If the scorer's LLM response is blocked by + content filtering. The transport only surfaces the condition; the calling + ``Scorer`` owns the policy for whether to raise or return a default score. + EmptyResponseException: If the scorer's LLM response contains no text piece to parse + and was not blocked (e.g. an empty or malformed response). InvalidJsonException: If the response is not valid JSON, is missing required keys, or fails the handler's value validation. Exception: For other unexpected errors during scoring. @@ -129,8 +134,29 @@ async def _run_llm_scoring_async( except Exception as ex: raise Exception(f"Error scoring prompt with original prompt ID: {scored_prompt_id}") from ex - # Get the text piece which contains the JSON response containing the score_value and rationale from the LLM - text_piece = next(piece for piece in response[0].message_pieces if piece.converted_value_data_type == "text") + # Resolve the text piece that holds the JSON response (score_value + rationale). When it's + # absent the scorer produced no parseable output. Guard on the actual failure (no text + # piece) rather than "all pieces blocked" so every no-text shape is handled instead of + # only the fully-blocked one. A content-filter block surfaces as a dedicated exception + # (the calling Scorer owns whether to raise or fall back); any other no-text response is a + # genuine empty/malformed error. Neither is retried by @pyrit_json_retry. + text_piece = next( + (piece for piece in response[0].message_pieces if piece.converted_value_data_type == "text"), None + ) + if text_piece is None: + if any(piece.is_blocked() for piece in response[0].message_pieces): + raise ScorerLLMResponseBlockedException( + message=( + f"The scorer's LLM response was blocked by content filtering while scoring " + f"prompt ID: {scored_prompt_id}. Consider using a scorer endpoint with " + f"content filtering disabled for red-teaming workflows." + ) + ) + raise EmptyResponseException( + message=( + f"The scorer's LLM response contained no text to parse while scoring prompt ID: {scored_prompt_id}." + ) + ) return response_handler.parse( response_text=text_piece.converted_value, diff --git a/pyrit/score/scorer.py b/pyrit/score/scorer.py index 226eb44cd8..30f0d86047 100644 --- a/pyrit/score/scorer.py +++ b/pyrit/score/scorer.py @@ -14,7 +14,7 @@ cast, ) -from pyrit.exceptions import PyritException +from pyrit.exceptions import PyritException, ScorerLLMResponseBlockedException from pyrit.memory import CentralMemory, MemoryInterface from pyrit.models import ( ChatMessageRole, @@ -81,6 +81,15 @@ class Scorer(Identifiable, abc.ABC): #: (Chat Completions API) and ``OpenAIResponseTarget`` (Responses API). score_blocked_content: bool = False + #: Controls what happens when the scorer's *own* LLM response is blocked by content + #: filtering (common in red-teaming, since the scorer's rationale quotes harmful content). + #: When True (default), scoring raises ``ScorerLLMResponseBlockedException`` — a blocked + #: scorer endpoint is treated as a real error. When False, scoring returns the scorer's + #: type default instead (False for true/false scorers, 0.0 for float-scale). This is + #: distinct from ``score_blocked_content``, which concerns the target-under-test response. + #: Set this on scorer instances before use. Defaults to True. + raise_if_scorer_blocks: bool = True + def __init_subclass__(cls, **kwargs: Any) -> None: """ Enforce the keyword-only constructor contract on subclasses. @@ -228,6 +237,8 @@ async def score_async( list[Score]: A list of Score objects representing the results. Raises: + ScorerLLMResponseBlockedException: If the scorer's own LLM response is blocked by + content filtering and ``raise_if_scorer_blocks`` is True (the default). PyritException: If scoring raises a PyRIT exception (re-raised with enhanced context). RuntimeError: If scoring raises a non-PyRIT exception (wrapped with scorer context). """ @@ -259,6 +270,25 @@ async def score_async( scoring_message, objective=objective, ) + except ScorerLLMResponseBlockedException as e: + # The scorer's own LLM response was content-filtered. By default this is a real + # error and re-raised; when raise_if_scorer_blocks is False, fall back to the + # scorer's type default (False / 0.0) instead. The decision lives here in the + # Scorer, not the transport (see doc/code/framework.md). + if self.raise_if_scorer_blocks: + e.message = f"Error in scorer {self.__class__.__name__}: {e.message}" + e.args = (f"Status Code: {e.status_code}, Message: {e.message}",) + raise + logger.info( + "Scorer %s LLM response was blocked by content filtering; " + "returning default score (raise_if_scorer_blocks=False).", + self.__class__.__name__, + ) + scores = self._build_fallback_score( + message=scoring_message, + objective=objective, + scorer_response_blocked=True, + ) except PyritException as e: # Re-raise PyRIT exceptions with enhanced context while preserving type for retry decorators e.message = f"Error in scorer {self.__class__.__name__}: {e.message}" @@ -401,7 +431,9 @@ def _get_supported_pieces(self, message: Message) -> list[MessagePiece]: ] @abstractmethod - def _build_fallback_score(self, *, message: Message, objective: str | None) -> list[Score]: + def _build_fallback_score( + self, *, message: Message, objective: str | None, scorer_response_blocked: bool = False + ) -> list[Score]: """ Return neutral fallback ``Score`` objects when ``_score_async`` produced no scores. @@ -420,6 +452,9 @@ def _build_fallback_score(self, *, message: Message, objective: str | None) -> l Args: message (Message): The (possibly substituted) message that was scored. objective (str | None): The objective associated with this scoring call. + scorer_response_blocked (bool): When True, the fallback was triggered because the + scorer's *own* LLM response was blocked by content filtering (not the + target-under-test). Subclasses should reflect this in the rationale. Returns: list[Score]: One or more fallback scores. Must not be empty. diff --git a/pyrit/score/true_false/true_false_scorer.py b/pyrit/score/true_false/true_false_scorer.py index 30f4ab21d5..c3bb3399a7 100644 --- a/pyrit/score/true_false/true_false_scorer.py +++ b/pyrit/score/true_false/true_false_scorer.py @@ -160,7 +160,9 @@ async def _score_async(self, message: Message, *, objective: str | None = None) ) ] - def _build_fallback_score(self, *, message: Message, objective: str | None) -> list[Score]: + def _build_fallback_score( + self, *, message: Message, objective: str | None, scorer_response_blocked: bool = False + ) -> list[Score]: """ Build a single-element list containing a ``false`` score when no pieces could be scored. @@ -170,6 +172,8 @@ def _build_fallback_score(self, *, message: Message, objective: str | None) -> l Args: message (Message): The message whose first piece is inspected for status. objective (str | None): The objective associated with this scoring call. + scorer_response_blocked (bool): When True, the scorer's own LLM response was + blocked by content filtering; reflected in the rationale. Returns: list[Score]: A single-element list containing a ``false`` ``true_false`` score @@ -183,7 +187,13 @@ def _build_fallback_score(self, *, message: Message, objective: str | None) -> l if piece_id is None: raise ValueError("Cannot create score: message piece has no id or original_prompt_id") - if first_piece.is_blocked(): + if scorer_response_blocked: + rationale = ( + "The scorer's own LLM response was blocked by content filtering " + "(raise_if_scorer_blocks is False); returning false." + ) + description = "Scorer response blocked; returning false." + elif first_piece.is_blocked(): rationale = ( "The request was blocked by the target " "(score_blocked_content is False or no partial content available); returning false." diff --git a/tests/unit/score/test_conversation_history_scorer.py b/tests/unit/score/test_conversation_history_scorer.py index ddfa663c79..0fa339b40e 100644 --- a/tests/unit/score/test_conversation_history_scorer.py +++ b/tests/unit/score/test_conversation_history_scorer.py @@ -68,7 +68,9 @@ async def _score_piece_async(self, message_piece: MessagePiece, *, objective: st def validate_return_scores(self, scores: list[Score]): pass - def _build_fallback_score(self, *, message: Message, objective: str | None) -> list[Score]: + def _build_fallback_score( + self, *, message: Message, objective: str | None, scorer_response_blocked: bool = False + ) -> list[Score]: return [ Score( score_value="false", diff --git a/tests/unit/score/test_scorer.py b/tests/unit/score/test_scorer.py index 741c41135b..2acde6de7c 100644 --- a/tests/unit/score/test_scorer.py +++ b/tests/unit/score/test_scorer.py @@ -14,6 +14,7 @@ from pyrit.models import ComponentIdentifier, Message, MessagePiece, Score from pyrit.prompt_target import PromptTarget from pyrit.score import ( + FloatScaleScorer, Scorer, ScorerPromptValidator, TrueFalseScorer, @@ -143,7 +144,9 @@ def validate_return_scores(self, scores: list[Score]): for score in scores: assert 0 <= float(score.score_value) <= 1 - def _build_fallback_score(self, *, message: Message, objective: str | None) -> list[Score]: + def _build_fallback_score( + self, *, message: Message, objective: str | None, scorer_response_blocked: bool = False + ) -> list[Score]: return [ Score( score_value="0.0", @@ -1550,6 +1553,246 @@ async def test_score_value_with_llm_skips_reasoning_piece(good_json): assert result.score_rationale == "Valid response" +async def test_score_value_with_llm_raises_when_scorer_response_blocked(): + """When the scorer's own LLM response is blocked, the transport raises ScorerLLMResponseBlockedException.""" + from pyrit.exceptions import ScorerLLMResponseBlockedException + + chat_target = MagicMock(PromptTarget) + chat_target.get_identifier.return_value = get_mock_target_identifier("MockChatTarget") + + blocked_piece = MessagePiece( + role="assistant", + original_value="", + original_value_data_type="error", + converted_value="", + converted_value_data_type="error", + conversation_id="test-convo", + response_error="blocked", + ) + blocked_response = Message(message_pieces=[blocked_piece]) + chat_target.send_prompt_async = AsyncMock(return_value=[blocked_response]) + + scorer = MockScorer() + + with pytest.raises(ScorerLLMResponseBlockedException, match="blocked by content filtering"): + await scorer._score_value_with_llm_async( + prompt_target=chat_target, + system_prompt="system_prompt", + message_value="message_value", + message_data_type="text", + scored_prompt_id="test-prompt-id", + category="category", + objective="task", + ) + + # A blocked response is a terminal condition, not a transient JSON error: it must not retry. + assert chat_target.send_prompt_async.call_count == 1 + + +async def test_score_value_with_llm_raises_empty_response_when_no_text_piece(): + """A no-text response that wasn't content-filtered raises EmptyResponseException, not blocked.""" + from pyrit.exceptions import EmptyResponseException + + chat_target = MagicMock(PromptTarget) + chat_target.get_identifier.return_value = get_mock_target_identifier("MockChatTarget") + + # An error piece that is NOT flagged as blocked (e.g. a flaky/empty response) and no text piece. + non_text_piece = MessagePiece( + role="assistant", + original_value="", + original_value_data_type="error", + converted_value="", + converted_value_data_type="error", + conversation_id="test-convo", + response_error="unknown", + ) + chat_target.send_prompt_async = AsyncMock(return_value=[Message(message_pieces=[non_text_piece])]) + + scorer = MockScorer() + + with pytest.raises(EmptyResponseException, match="no text to parse"): + await scorer._score_value_with_llm_async( + prompt_target=chat_target, + system_prompt="system_prompt", + message_value="message_value", + message_data_type="text", + scored_prompt_id="test-prompt-id", + category="category", + objective="task", + ) + + # No parseable text is terminal here, not a transient JSON error: it must not retry. + assert chat_target.send_prompt_async.call_count == 1 + + +# ── Axis B: the scorer's own LLM response is blocked (raise_if_scorer_blocks) ───────────── + + +class _ForwarderTrueFalseScorer(TrueFalseScorer): + """TrueFalseScorer whose piece scoring goes through the base ``_score_value_with_llm_async`` forwarder.""" + + def __init__(self, *, chat_target: PromptTarget) -> None: + super().__init__(validator=DummyValidator()) + self._prompt_target = chat_target + self._system_prompt = "system" + + def _build_identifier(self) -> ComponentIdentifier: + return self._create_identifier() + + async def _score_piece_async(self, message_piece: MessagePiece, *, objective: str | None = None) -> list[Score]: + unvalidated = await self._score_value_with_llm_async( + prompt_target=self._prompt_target, + system_prompt=self._system_prompt, + message_value=message_piece.converted_value, + message_data_type="text", + scored_prompt_id=message_piece.id, + objective=objective, + ) + return [unvalidated.to_score(score_value=unvalidated.raw_score_value, score_type="true_false")] + + +class _DirectTransportTrueFalseScorer(TrueFalseScorer): + """TrueFalseScorer that calls ``_run_llm_scoring_async`` directly, like SelfAskTrueFalseScorer.""" + + def __init__(self, *, chat_target: PromptTarget) -> None: + from pyrit.score import JsonSchemaResponseHandler + + super().__init__(validator=DummyValidator()) + self._prompt_target = chat_target + self._system_prompt = "system" + self._response_handler = JsonSchemaResponseHandler() + + def _build_identifier(self) -> ComponentIdentifier: + return self._create_identifier() + + async def _score_piece_async(self, message_piece: MessagePiece, *, objective: str | None = None) -> list[Score]: + from pyrit.score.llm_scoring import _run_llm_scoring_async + + unvalidated = await _run_llm_scoring_async( + chat_target=self._prompt_target, + system_prompt=self._system_prompt, + response_handler=self._response_handler, + value=message_piece.converted_value, + data_type="text", + scored_prompt_id=message_piece.id, + scorer_identifier=self.get_identifier(), + objective=objective, + ) + return [unvalidated.to_score(score_value=unvalidated.raw_score_value, score_type="true_false")] + + +class _ForwarderFloatScaleScorer(FloatScaleScorer): + """FloatScaleScorer whose piece scoring goes through the base ``_score_value_with_llm_async`` forwarder.""" + + def __init__(self, *, chat_target: PromptTarget) -> None: + super().__init__(validator=DummyValidator()) + self._prompt_target = chat_target + self._system_prompt = "system" + + def _build_identifier(self) -> ComponentIdentifier: + return self._create_identifier() + + async def _score_piece_async(self, message_piece: MessagePiece, *, objective: str | None = None) -> list[Score]: + unvalidated = await self._score_value_with_llm_async( + prompt_target=self._prompt_target, + system_prompt=self._system_prompt, + message_value=message_piece.converted_value, + message_data_type="text", + scored_prompt_id=message_piece.id, + objective=objective, + numeric_value=True, + ) + return [unvalidated.to_score(score_value=unvalidated.raw_score_value, score_type="float_scale")] + + +def _make_scorer_blocking_target() -> MagicMock: + """A chat target mock whose response is fully blocked by content filtering.""" + chat_target = MagicMock(PromptTarget) + chat_target.get_identifier.return_value = get_mock_target_identifier("MockChatTarget") + chat_target.set_system_prompt = MagicMock() + blocked_piece = MessagePiece( + role="assistant", + original_value="", + original_value_data_type="error", + converted_value="", + converted_value_data_type="error", + conversation_id="scorer-convo", + response_error="blocked", + ) + chat_target.send_prompt_async = AsyncMock(return_value=[Message(message_pieces=[blocked_piece])]) + return chat_target + + +def _make_normal_input_message() -> Message: + """A normal (non-blocked) message to be scored.""" + return Message( + message_pieces=[ + MessagePiece( + role="assistant", + original_value="some response to score", + converted_value="some response to score", + original_value_data_type="text", + converted_value_data_type="text", + conversation_id="input-convo", + ) + ] + ) + + +@pytest.mark.usefixtures("patch_central_database") +class TestScorerResponseBlocked: + """Axis B: behavior when the scorer's own LLM response is content-filtered.""" + + async def test_raises_by_default(self): + from pyrit.exceptions import ScorerLLMResponseBlockedException + + scorer = _ForwarderTrueFalseScorer(chat_target=_make_scorer_blocking_target()) + + with pytest.raises(ScorerLLMResponseBlockedException, match="blocked by content filtering"): + await scorer.score_async(_make_normal_input_message()) + + async def test_returns_false_when_flag_disabled(self): + target = _make_scorer_blocking_target() + scorer = _ForwarderTrueFalseScorer(chat_target=target) + scorer.raise_if_scorer_blocks = False + + scores = await scorer.score_async(_make_normal_input_message()) + + assert len(scores) == 1 + assert scores[0].score_value == "false" + assert "blocked by content filtering" in scores[0].score_rationale + # Blocked is terminal: no retry storm. + assert target.send_prompt_async.call_count == 1 + + async def test_returns_zero_for_float_scale_when_flag_disabled(self): + scorer = _ForwarderFloatScaleScorer(chat_target=_make_scorer_blocking_target()) + scorer.raise_if_scorer_blocks = False + + scores = await scorer.score_async(_make_normal_input_message()) + + assert len(scores) == 1 + assert scores[0].score_value == "0.0" + assert "blocked by content filtering" in scores[0].score_rationale + + async def test_direct_transport_caller_raises_by_default(self): + from pyrit.exceptions import ScorerLLMResponseBlockedException + + scorer = _DirectTransportTrueFalseScorer(chat_target=_make_scorer_blocking_target()) + + with pytest.raises(ScorerLLMResponseBlockedException, match="blocked by content filtering"): + await scorer.score_async(_make_normal_input_message()) + + async def test_direct_transport_caller_returns_false_when_flag_disabled(self): + scorer = _DirectTransportTrueFalseScorer(chat_target=_make_scorer_blocking_target()) + scorer.raise_if_scorer_blocks = False + + scores = await scorer.score_async(_make_normal_input_message()) + + assert len(scores) == 1 + assert scores[0].score_value == "false" + assert "blocked by content filtering" in scores[0].score_rationale + + # ── Helpers for score_blocked_content tests ──────────────────────────────────