FEAT: Add GitHub Copilot SDK target and native judge retries

pyproject.toml

@@ -135,12 +135,17 @@ litellm = [
     "litellm>=1.84.0",
 ]
 
+github-copilot = [
+    "github-copilot-sdk>=1.0.11",
+]
+
 # all includes all functional dependencies excluding the ones from the "dev" dependency group
 all = [
     "accelerate>=1.7.0",
     "azure-ai-ml>=1.32.0",
     "azure-cognitiveservices-speech>=1.44.0",
     "flask>=3.1.3",
+    "github-copilot-sdk>=1.0.11",
     "ipykernel>=6.29.5",
     "jupyter>=1.1.1",
     "litellm>=1.84.0",

pyrit/prompt_target/__init__.py

@@ -31,6 +31,7 @@
     from pyrit.prompt_target.common.target_requirements import CHAT_TARGET_REQUIREMENTS, TargetRequirements
     from pyrit.prompt_target.common.utils import limit_requests_per_minute
     from pyrit.prompt_target.gandalf_target import GandalfLevel, GandalfTarget
+    from pyrit.prompt_target.github_copilot_target import GitHubCopilotTarget
     from pyrit.prompt_target.http_target.http_target import HTTPTarget
     from pyrit.prompt_target.http_target.http_target_callback_functions import (
         get_http_target_json_response_callback_function,
@@ -66,6 +67,7 @@
     "ConversationNormalizationPipeline": "pyrit.prompt_target.common.conversation_normalization_pipeline",
     "GandalfLevel": "pyrit.prompt_target.gandalf_target",
     "GandalfTarget": "pyrit.prompt_target.gandalf_target",
+    "GitHubCopilotTarget": "pyrit.prompt_target.github_copilot_target",
     "get_http_target_json_response_callback_function": "pyrit.prompt_target.http_target.http_target_callback_functions",
     "get_http_target_regex_matching_callback_function": (
         "pyrit.prompt_target.http_target.http_target_callback_functions"

pyrit/prompt_target/common/prompt_target.py

@@ -334,13 +334,16 @@ def set_system_prompt(
             conversation_id (str): The conversation id to attach the prompt to.
 
         Raises:
-            ValueError: If the target does not support multi-turn or editable history.
+            ValueError: If the target does not support multi-turn conversations, or
+                supports neither editable history nor native system prompts.
             RuntimeError: If the conversation already has messages.
         """
-        if not self.capabilities.supports_multi_turn or not self.capabilities.supports_editable_history:
+        if not self.capabilities.supports_multi_turn or not (
+            self.capabilities.supports_editable_history or self.capabilities.supports_system_prompt
+        ):
             raise ValueError(
                 f"Target {type(self).__name__} does not support setting a system prompt. "
-                "It must support both multi-turn conversations and editable history."
+                "It must support multi-turn conversations and either editable history or native system prompts."
             )
 
         messages = self._memory.get_conversation_messages(conversation_id=conversation_id)

pyrit/score/llm_scoring.py

@@ -13,6 +13,7 @@
     EmptyResponseException,
     InvalidJsonException,
     ScorerLLMResponseBlockedException,
+    pyrit_json_retry,
 )
 from pyrit.models import (
     Acquisition,
@@ -83,6 +84,7 @@ async def _run_llm_scoring_async(
     category: Sequence[str] | str | None = None,
     objective: str | None = None,
     normalizer: PromptNormalizer | None = None,
+    fresh_conversation_per_attempt: bool = False,
     observation_metadata: Mapping[str, str] | None = None,
     requires_message_piece_evidence: bool = False,
     judgment_replay_identifier: Mapping[str, object] | None = None,
@@ -93,13 +95,13 @@ async def _run_llm_scoring_async(
     This is the shared LLM evaluation mechanism: it optionally sets a system prompt on the target, sends
     the value to be scored (forwarding ``response_handler.json_response_config`` so targets that
     support structured output can enforce it), and delegates parsing and validation to
-    ``response_handler``. The round-trip is routed through a ``PromptNormalizer`` via
-    ``send_json_with_retry_async`` so the scorer's question and the target's answer are persisted
-    to memory (a full audit trail, and a real conversation an attack can link as a SCORE-type
-    related conversation) and so JSON retries roll memory back to a clean baseline between attempts
-    instead of replaying the target's own malformed reply. It is intentionally stateless and
-    independent of any particular ``Scorer`` so that scorers can compose it without inheriting LLM
-    machinery.
+    ``response_handler``. The round-trip uses a ``PromptNormalizer`` so the scorer's question and
+    the target's answer are persisted to memory (a full audit trail, and a real conversation an
+    attack can link as a SCORE-type related conversation). The default editable-history path rolls
+    memory back between JSON attempts; ``fresh_conversation_per_attempt`` keeps malformed judge
+    exchanges in separate conversations instead of replaying native history. It is intentionally
+    stateless and independent of any particular ``Scorer`` so scorers can compose it without
+    inheriting LLM machinery.
 
     The round-trip owns only the transport; the ``ResponseHandler`` owns the response contract —
     the optional response schema and turning raw text into a validated ``UnvalidatedScore``.
@@ -127,8 +129,10 @@ async def _run_llm_scoring_async(
         objective (str | None): Transitional objective context for direct helper callers.
             Defaults to None.
         normalizer (PromptNormalizer | None): Normalizer used to send the scoring round-trip
-            and whose memory is rolled back between JSON retries. Injectable for testing;
-            defaults to a fresh ``PromptNormalizer()`` when not supplied.
+            and resolve scorer evidence. Injectable for testing; defaults to a fresh
+            ``PromptNormalizer()`` when not supplied.
+        fresh_conversation_per_attempt (bool): Use a new conversation for each JSON retry when
+            target history cannot be rolled back. Defaults to False.
         observation_metadata (Mapping[str, str] | None): Scorer-specific state required to
             reconstruct the response parser during replay. Defaults to None.
         requires_message_piece_evidence (bool): Whether the rendered request reads fields that a
@@ -193,7 +197,7 @@ async def _run_llm_scoring_async(
         observation_scorable is not None and scored_evidence_digest is not None and has_required_evidence
     )
 
-    if system_prompt is not None:
+    if system_prompt is not None and not fresh_conversation_per_attempt:
         chat_target.set_system_prompt(
             system_prompt=system_prompt,
             conversation_id=conversation_id,
@@ -273,17 +277,51 @@ def _parse(response: Message) -> UnvalidatedScore:
             objective=expectation.objective if expectation else None,
         )
 
-    # Route the round-trip through the normalizer so the scorer Q&A is persisted and JSON retries
-    # replay on a clean history.
+    # Editable targets retry on a rolled-back conversation; non-editable judges opt into fresh sessions.
     try:
-        unvalidated_score = await send_json_with_retry_async(
-            normalizer=resolved_normalizer,
-            target=chat_target,
-            message=scorer_llm_request,
-            conversation_id=conversation_id,
-            parse=_parse,
-            on_response=_capture_response,
-        )
+        if fresh_conversation_per_attempt:
+            first_attempt = True
+
+            @pyrit_json_retry
+            async def _fresh_attempt_async() -> UnvalidatedScore:
+                nonlocal first_attempt
+                attempt_conversation_id = conversation_id if first_attempt else str(uuid.uuid4())
+                attempt_message = scorer_llm_request if first_attempt else scorer_llm_request.duplicate()
+                if system_prompt is not None:
+                    chat_target.set_system_prompt(
+                        system_prompt=system_prompt,
+                        conversation_id=attempt_conversation_id,
+                    )
+                first_attempt = False
+                response = await resolved_normalizer.send_prompt_async(
+                    message=attempt_message,
+                    conversation_id=attempt_conversation_id,
+                    target=chat_target,
+                )
+                if not response:
+                    raise ValueError(f"No response received for conversation ID: {attempt_conversation_id}")
+                _capture_response(response)
+                try:
+                    return _parse(response)
+                except InvalidJsonException:
+                    try:
+                        await chat_target.reset_conversation_async(conversation_id=attempt_conversation_id)
+                    except Exception as reset_error:
+                        raise RuntimeError(
+                            "Could not release the malformed judge session; refusing another retry."
+                        ) from reset_error
+                    raise
+
+            unvalidated_score: UnvalidatedScore = await _fresh_attempt_async()
+        else:
+            unvalidated_score = await send_json_with_retry_async(
+                normalizer=resolved_normalizer,
+                target=chat_target,
+                message=scorer_llm_request,
+                conversation_id=conversation_id,
+                parse=_parse,
+                on_response=_capture_response,
+            )
     except ScorerLLMResponseBlockedException as error:
         if terminal_response is not None and can_collect_observation and _has_observation_collection():
             observation = _build_judgment_observation(

pyrit/score/scorer.py

@@ -7,6 +7,7 @@
 import asyncio
 import logging
 from abc import abstractmethod
+from dataclasses import replace
 from typing import TYPE_CHECKING, Any, ClassVar, TypeVar, cast, final, overload
 
 from pyrit.common.deprecation import print_deprecation_message
@@ -32,7 +33,8 @@
     ScoringExpectation,
 )
 from pyrit.prompt_target.batch_helper import batch_task_async
-from pyrit.prompt_target.common.target_requirements import TargetRequirements
+from pyrit.prompt_target.common.target_capabilities import CapabilityName
+from pyrit.prompt_target.common.target_requirements import CHAT_TARGET_REQUIREMENTS, TargetRequirements
 from pyrit.score.observation.execution import (
     NonReplayableObservationError,
     _observation_collection,
@@ -62,6 +64,18 @@
 ConditionT = TypeVar("ConditionT", bound=Condition)
 
 
+class _SelfContainedJudgeTargetRequirements(TargetRequirements):
+    def validate(self, *, target: PromptTarget) -> None:
+        requirements = CHAT_TARGET_REQUIREMENTS
+        if not target.capabilities.supports_editable_history:
+            requirements = replace(
+                requirements,
+                required=requirements.required - {CapabilityName.EDITABLE_HISTORY},
+                native_required=requirements.native_required | {CapabilityName.SYSTEM_PROMPT},
+            )
+        requirements.validate(target=target)
+
+
 async def _legacy_score_scorable_async(
     self: Scorer,
     *,

pyrit/score/true_false/self_ask_question_answer_scorer.py

@@ -7,6 +7,7 @@
 
 from pyrit.common.path import SCORER_SEED_PROMPT_PATH
 from pyrit.models import AnswerMatches, ScoringExpectation
+from pyrit.prompt_target import CHAT_TARGET_REQUIREMENTS
 from pyrit.score.llm_scoring import _run_llm_scoring_async
 from pyrit.score.scorer_prompt_validator import ScorerPromptValidator
 from pyrit.score.true_false.self_ask_true_false_scorer import (
@@ -52,6 +53,7 @@ class SelfAskQuestionAnswerScorer(SelfAskTrueFalseScorer):
     _JUDGMENT_PROMPT = (
         "{objective}\n\nThe target response is as follows. Evaluate if the response is correct:\n{response}"
     )
+    TARGET_REQUIREMENTS = CHAT_TARGET_REQUIREMENTS
 
     def __init__(
         self,

pyrit/score/true_false/self_ask_refusal_scorer.py

@@ -17,10 +17,11 @@
     ScoringExpectation,
     SeedPrompt,
 )
-from pyrit.prompt_target import CHAT_TARGET_REQUIREMENTS, PromptTarget
+from pyrit.prompt_target import PromptTarget
 from pyrit.score.llm_scoring import _parse_judgment_observation, _run_llm_scoring_async
 from pyrit.score.observation.execution import _ObservationEvidence
 from pyrit.score.response_handler import JsonSchemaResponseHandler, ResponseHandler, TrueFalseResponseHandler
+from pyrit.score.scorer import _SelfContainedJudgeTargetRequirements
 from pyrit.score.scorer_prompt_validator import ScorerPromptValidator
 from pyrit.score.true_false.true_false_score_aggregator import (
     TrueFalseAggregatorFunc,
@@ -76,7 +77,7 @@ class SelfAskRefusalScorer(MessageTrueFalseScorer):
     )
 
     _DEFAULT_VALIDATOR: ScorerPromptValidator = ScorerPromptValidator()
-    TARGET_REQUIREMENTS = CHAT_TARGET_REQUIREMENTS
+    TARGET_REQUIREMENTS = _SelfContainedJudgeTargetRequirements()
 
     def __init__(
         self,
@@ -93,8 +94,9 @@ def __init__(
         Initialize the SelfAskRefusalScorer.
 
         Args:
-            chat_target (PromptTarget | None): The chat target used for scoring. Must satisfy
-                CHAT_TARGET_REQUIREMENTS.
+            chat_target (PromptTarget | None): The chat target used for scoring. Must support
+                multi-turn conversations and either editable history or native system prompts.
+                Non-editable targets use fresh conversations when malformed JSON is retried.
             system_prompt (SeedPrompt | str | None): The refusal-detection system prompt. A
                 ``SeedPrompt`` (e.g. loaded from a ``RefusalScorerPaths`` YAML) is used verbatim and
                 may carry a ``response_json_schema``; a ``str`` is used as-is; ``None`` falls back to
@@ -248,6 +250,7 @@ async def _score_piece_async(self, message_piece: MessagePiece, *, objective: st
             scorer_identifier=self.get_identifier(),
             judgment_replay_identifier=self._get_judgment_replay_identifier(),
             category=self._score_category,
+            fresh_conversation_per_attempt=not self._prompt_target.capabilities.supports_editable_history,
         )
         score = unvalidated_score.to_score(score_value=unvalidated_score.raw_score_value, score_type="true_false")
 

pyrit/score/true_false/self_ask_true_false_scorer.py

@@ -20,10 +20,11 @@
     SeedPrompt,
     UnvalidatedScore,
 )
-from pyrit.prompt_target import CHAT_TARGET_REQUIREMENTS, PromptTarget
+from pyrit.prompt_target import PromptTarget
 from pyrit.score.llm_scoring import _parse_judgment_observation, _run_llm_scoring_async
 from pyrit.score.observation.execution import _ObservationEvidence
 from pyrit.score.response_handler import JsonSchemaResponseHandler, ResponseHandler, TrueFalseResponseHandler
+from pyrit.score.scorer import _SelfContainedJudgeTargetRequirements
 from pyrit.score.scorer_prompt_validator import ScorerPromptValidator
 from pyrit.score.system_prompt import _render_system_prompt_template
 from pyrit.score.true_false.true_false_score_aggregator import (
@@ -153,7 +154,7 @@ class SelfAskTrueFalseScorer(MessageTrueFalseScorer):
     _DEFAULT_VALIDATOR: ScorerPromptValidator = ScorerPromptValidator(
         supported_data_types=["text", "image_path"],
     )
-    TARGET_REQUIREMENTS = CHAT_TARGET_REQUIREMENTS
+    TARGET_REQUIREMENTS = _SelfContainedJudgeTargetRequirements()
 
     def __init__(
         self,
@@ -169,8 +170,9 @@ def __init__(
         Initialize the SelfAskTrueFalseScorer.
 
         Args:
-            chat_target (PromptTarget | None): The chat target used for scoring. Must satisfy
-                CHAT_TARGET_REQUIREMENTS.
+            chat_target (PromptTarget | None): The chat target used for scoring. Must support multi-turn
+                conversations and either editable history or native system prompts. Noneditable targets
+                are supported for text scoring only.
             system_prompt (SeedPrompt | str | None): The scoring system prompt. A ``SeedPrompt``
                 (e.g. rendered via ``render_true_false_system_prompt``) is used verbatim and may
                 carry a ``response_json_schema``; a ``str`` is used as-is; ``None`` falls back to the
@@ -305,9 +307,17 @@ async def _score_piece_async(self, message_piece: MessagePiece, *, objective: st
                 The category is configured from the TrueFalseQuestionPath.
                 The score_value is True or False based on which description fits best.
                 Metadata can be configured to provide additional information.
+
+        Raises:
+            ValueError: If non-text scoring uses a target without editable history.
         """
         # Build scoring prompt - for non-text content, extra context about objective is sent as a prepended text piece
         is_non_text = message_piece.converted_value_data_type != "text"
+        if is_non_text and not self._prompt_target.capabilities.supports_editable_history:
+            raise ValueError(
+                "non-text scoring requires editable history; fresh-conversation retries support text only."
+            )
+
         if is_non_text:
             prepended_text = f"objective: {objective}\nresponse:"
             scoring_value = message_piece.converted_value
@@ -328,6 +338,9 @@ async def _score_piece_async(self, message_piece: MessagePiece, *, objective: st
             judgment_replay_identifier=self._get_judgment_replay_identifier(),
             prepended_text=prepended_text,
             category=self._score_category,
+            fresh_conversation_per_attempt=(
+                scoring_data_type == "text" and not self._prompt_target.capabilities.supports_editable_history
+            ),
         )
 
         return [self._convert_score(unvalidated_score)]

tests/unit/mocks.py

@@ -205,8 +205,14 @@ class MockPromptTarget(PromptTarget):
 
     prompt_sent: list[str]
 
-    def __init__(self, *, id=None, rpm=None) -> None:  # noqa: A002
-        super().__init__(max_requests_per_minute=rpm)
+    def __init__(
+        self,
+        *,
+        id=None,  # noqa: A002
+        rpm=None,
+        custom_configuration: TargetConfiguration | None = None,
+    ) -> None:
+        super().__init__(max_requests_per_minute=rpm, custom_configuration=custom_configuration)
         self.id = id
         self.prompt_sent = []