diff --git a/.github/instructions/converters.instructions.md b/.github/instructions/converters.instructions.md index 9c00bb2ed1..031832bb73 100644 --- a/.github/instructions/converters.instructions.md +++ b/.github/instructions/converters.instructions.md @@ -65,7 +65,7 @@ from pyrit.identifiers import ComponentIdentifier For LLM-based converters, also import: ```python -from pyrit.prompt_target import PromptChatTarget +from pyrit.prompt_target import PromptTarget ``` ## Constructor Pattern @@ -77,7 +77,7 @@ from pyrit.common.apply_defaults import apply_defaults class MyConverter(PromptConverter): @apply_defaults - def __init__(self, *, target: PromptChatTarget, template: str = "default") -> None: + def __init__(self, *, target: PromptTarget, template: str = "default") -> None: ... ``` diff --git a/.github/instructions/scenarios.instructions.md b/.github/instructions/scenarios.instructions.md index 93fe20fed4..9261dca942 100644 --- a/.github/instructions/scenarios.instructions.md +++ b/.github/instructions/scenarios.instructions.md @@ -41,7 +41,7 @@ class MyScenario(Scenario): def __init__( self, *, - adversarial_chat: PromptChatTarget | None = None, + adversarial_chat: PromptTarget | None = None, objective_scorer: TrueFalseScorer | None = None, scenario_result_id: str | None = None, ) -> None: diff --git a/doc/code/datasets/5_simulated_conversation.ipynb b/doc/code/datasets/5_simulated_conversation.ipynb index 920570602e..113b421fc7 100644 --- a/doc/code/datasets/5_simulated_conversation.ipynb +++ b/doc/code/datasets/5_simulated_conversation.ipynb @@ -544,7 +544,7 @@ "| Parameter | Type | Description |\n", "|-----------|------|-------------|\n", "| `objective` | `str` | The goal the adversarial chat works toward |\n", - "| `adversarial_chat` | `PromptChatTarget` | The LLM that generates attack prompts (also plays the simulated target) |\n", + "| `adversarial_chat` | `PromptTarget` | The LLM that generates attack prompts (also plays the simulated target). Must declare `supports_multi_turn=True` and `supports_editable_history=True`. |\n", "| `objective_scorer` | `TrueFalseScorer` | Evaluates whether the final turn achieved the objective |\n", "| `num_turns` | `int` | Number of conversation turns to generate (default: 3) |\n", "| `adversarial_chat_system_prompt_path` | `str \\| Path` | System prompt for the adversarial chat role |\n", diff --git a/doc/code/datasets/5_simulated_conversation.py b/doc/code/datasets/5_simulated_conversation.py index d01eb2da82..fb51ae7a8c 100644 --- a/doc/code/datasets/5_simulated_conversation.py +++ b/doc/code/datasets/5_simulated_conversation.py @@ -125,7 +125,7 @@ # | Parameter | Type | Description | # |-----------|------|-------------| # | `objective` | `str` | The goal the adversarial chat works toward | -# | `adversarial_chat` | `PromptChatTarget` | The LLM that generates attack prompts (also plays the simulated target) | +# | `adversarial_chat` | `PromptTarget` | The LLM that generates attack prompts (also plays the simulated target). Must declare `supports_multi_turn=True` and `supports_editable_history=True`. | # | `objective_scorer` | `TrueFalseScorer` | Evaluates whether the final turn achieved the objective | # | `num_turns` | `int` | Number of conversation turns to generate (default: 3) | # | `adversarial_chat_system_prompt_path` | `str \| Path` | System prompt for the adversarial chat role | diff --git a/doc/code/executor/attack/2_red_teaming_attack.ipynb b/doc/code/executor/attack/2_red_teaming_attack.ipynb index 4f3fb9bb38..a434402415 100644 --- a/doc/code/executor/attack/2_red_teaming_attack.ipynb +++ b/doc/code/executor/attack/2_red_teaming_attack.ipynb @@ -1456,7 +1456,7 @@ "source": [ "## Other Multi-Turn Attacks\n", "\n", - "The above examples should work using other multi-turn attacks with minimal modification. Check out attacks under `pyrit.executor.attack.multi_turn` for other examples, like Crescendo and Tree of Attacks. These algorithms are always more effective than `RedTeamingAttack`, which is a simple algorithm. However, `RedTeamingAttack` by its nature supports more targets - because it doesn't modify conversation history it can support any `PromptTarget` and not only `PromptChatTargets`." + " \"The above examples should work using other multi-turn attacks with minimal modification. Check out attacks under `pyrit.executor.attack.multi_turn` for other examples, like Crescendo and Tree of Attacks. These algorithms are always more effective than `RedTeamingAttack`, which is a simple algorithm. However, `RedTeamingAttack` by its nature supports more targets - because it doesn't modify conversation history it can support any `PromptTarget`, not only chat-style targets that declare `supports_multi_turn=True` and `supports_editable_history=True`.\"" ] } ], diff --git a/doc/code/executor/attack/2_red_teaming_attack.py b/doc/code/executor/attack/2_red_teaming_attack.py index b61e609c25..98d98773e7 100644 --- a/doc/code/executor/attack/2_red_teaming_attack.py +++ b/doc/code/executor/attack/2_red_teaming_attack.py @@ -314,4 +314,4 @@ # %% [markdown] # ## Other Multi-Turn Attacks # -# The above examples should work using other multi-turn attacks with minimal modification. Check out attacks under `pyrit.executor.attack.multi_turn` for other examples, like Crescendo and Tree of Attacks. These algorithms are always more effective than `RedTeamingAttack`, which is a simple algorithm. However, `RedTeamingAttack` by its nature supports more targets - because it doesn't modify conversation history it can support any `PromptTarget` and not only `PromptChatTargets`. +# The above examples should work using other multi-turn attacks with minimal modification. Check out attacks under `pyrit.executor.attack.multi_turn` for other examples, like Crescendo and Tree of Attacks. These algorithms are always more effective than `RedTeamingAttack`, which is a simple algorithm. However, `RedTeamingAttack` by its nature supports more targets - because it doesn't modify conversation history it can support any `PromptTarget`, not only chat-style targets that declare `supports_multi_turn=True` and `supports_editable_history=True`. diff --git a/doc/code/setup/default_values.md b/doc/code/setup/default_values.md index b8d36ff321..04efe67e24 100644 --- a/doc/code/setup/default_values.md +++ b/doc/code/setup/default_values.md @@ -23,7 +23,7 @@ from pyrit.common.apply_defaults import apply_defaults class MyConverter(PromptConverter): @apply_defaults - def __init__(self, *, converter_target: Optional[PromptChatTarget] = None, temperature: Optional[float] = None): + def __init__(self, *, converter_target: Optional[PromptTarget] = None, temperature: Optional[float] = None): self.converter_target = converter_target self.temperature = temperature ``` diff --git a/doc/code/targets/0_prompt_targets.md b/doc/code/targets/0_prompt_targets.md index 01a81cf9d3..e00983f769 100644 --- a/doc/code/targets/0_prompt_targets.md +++ b/doc/code/targets/0_prompt_targets.md @@ -19,20 +19,93 @@ async def send_prompt_async(self, *, message: Message) -> Message: A `Message` object is a normalized object with all the information a target will need to send a prompt, including a way to get a history for that prompt (in the cases that also needs to be sent). This is discussed in more depth [here](../memory/3_memory_data_types.md). -## PromptChatTargets vs PromptTargets +## Chat-style targets vs general targets A `PromptTarget` is a generic place to send a prompt. With PyRIT, the idea is that it will eventually be consumed by an AI application, but that doesn't have to be immediate. For example, you could have a SharePoint target. Everything you send a prompt to is a `PromptTarget`. Many attacks work generically with any `PromptTarget` including `RedTeamingAttack` and `PromptSendingAttack`. -With some algorithms, you want to send a prompt, set a system prompt, and modify conversation history (including PAIR [@chao2023pair], TAP [@mehrotra2023tap], and flip attack [@li2024flipattack]). These often require a `PromptChatTarget`, which implies you can modify a conversation history. `PromptChatTarget` is a subclass of `PromptTarget`. +With some algorithms, you want to send a prompt, set a system prompt, and modify conversation history (including PAIR [@chao2023pair], TAP [@mehrotra2023tap], and flip attack [@li2024flipattack]). These algorithms require a target whose [`TargetCapabilities`](#target-capabilities) declare both `supports_multi_turn=True` and `supports_editable_history=True` — i.e. you can modify a conversation history. Consumers express this requirement via `CHAT_TARGET_REQUIREMENTS` and validate it against `target.configuration` at construction time. See [Target Capabilities](#target-capabilities) below for the full list of capabilities and how they compose into a `TargetConfiguration`. + +Note: The previous `PromptChatTarget` class is **deprecated** as of v0.13.0 and will be removed in v0.15.0. Use `PromptTarget` directly with a `TargetConfiguration` declaring `supports_multi_turn=True` and `supports_editable_history=True`. See [Target Capabilities](#target-capabilities) for details. + Here are some examples: -| Example | Is `PromptChatTarget`? | Notes | -|-------------------------------------|---------------------------------------|-------------------------------------------------------------------------------------------------| -| **OpenAIChatTarget** (e.g., GPT-4) | **Yes** (`PromptChatTarget`) | Designed for conversational prompts (system messages, conversation history, etc.). | -| **OpenAIImageTarget** | **No** (not a `PromptChatTarget`) | Used for image generation; does not manage conversation history. | -| **HTTPTarget** | **No** (not a `PromptChatTarget`) | Generic HTTP target. Some apps might allow conversation history, but this target doesn't handle it. | -| **AzureBlobStorageTarget** | **No** (not a `PromptChatTarget`) | Used primarily for storage; not for conversation-based AI. | +| Example | Chat-style target? | Notes | +|-------------------------------------|---------------------------------------------------|-------------------------------------------------------------------------------------------------| +| **OpenAIChatTarget** (e.g., GPT-4) | **Yes** (multi-turn + editable history) | Designed for conversational prompts (system messages, conversation history, etc.). | +| **OpenAIImageTarget** | **No** | Used for image generation; does not manage conversation history. | +| **HTTPTarget** | **No** | Generic HTTP target. Some apps might allow conversation history, but this target doesn't handle it. | +| **AzureBlobStorageTarget** | **No** | Used primarily for storage; not for conversation-based AI. | + +## Target Capabilities + +Every `PromptTarget` exposes a `TargetConfiguration` (via `target.configuration`) that declares what the target natively supports. This lets attacks, converters, and scorers reason about whether a given target is suitable for a given workflow — and, where possible, adapt automatically when a capability is missing. + +A `TargetConfiguration` composes three concerns: + +- **`TargetCapabilities`** — an immutable, declarative description of what the target natively supports. +- **`CapabilityHandlingPolicy`** — for each capability that *can* be adapted, whether to `ADAPT` (apply a normalization step to work around the gap) or `RAISE` (fail immediately). +- **`ConversationNormalizationPipeline`** — the ordered set of normalizers derived from the gap between the declared capabilities and the policy. + +### Capabilities + +`TargetCapabilities` declares the following flags: + +| Capability | Meaning | +|----------------------------------|------------------------------------------------------------------------------------------------------------------------------------------| +| `supports_multi_turn` | The target accepts and uses conversation history (or maintains state externally, e.g. via a WebSocket). | +| `supports_multi_message_pieces` | The target accepts more than one `MessagePiece` in a single request (e.g. text + image in one message). | +| `supports_editable_history` | The conversation history can be modified after the fact. Implies `supports_multi_turn`. Required for attacks that rewrite prior turns. | +| `supports_system_prompt` | The target natively supports a system-role message. | +| `supports_json_output` | The target supports a "json" response format that guarantees valid JSON output. | +| `supports_json_schema` | The target supports constraining output to a caller-provided JSON schema. | +| `input_modalities` | The set of input modality combinations the target accepts (e.g. `{text}`, `{text, image_path}`). | +| `output_modalities` | The set of output modality combinations the target produces. | + +Each target class defines defaults; instances can override individual capabilities when they depend on deployment configuration (e.g. `HTTPTarget`, `PlaywrightTarget`). + +For well-known underlying models, you can look up a profile with `TargetCapabilities.get_known_capabilities(underlying_model="gpt-4o")`. + +### How consumers use capabilities + +Components that need a particular capability declare it as a `TargetRequirements` and validate at construction time: + +```python +from pyrit.prompt_target import CHAT_TARGET_REQUIREMENTS + +CHAT_TARGET_REQUIREMENTS.validate(target=target) +``` + +`TargetRequirements.validate` collects every missing capability and raises a single `ValueError`. For one-off checks against a single capability you can also call `target.configuration.ensure_can_handle(capability=...)` directly. + +### Adapting vs raising + +Some capability gaps can be papered over by PyRIT itself. For example, a single-turn target can be made to *appear* multi-turn by flattening the conversation history into a single prompt before sending. The `CapabilityHandlingPolicy` controls this on a per-capability basis: + +- `UnsupportedCapabilityBehavior.RAISE` — fail at construction time. This is the safe default. +- `UnsupportedCapabilityBehavior.ADAPT` — run the corresponding normalizer in the conversation pipeline before the target sees the messages. + +Non-adaptable capabilities (e.g. `supports_editable_history`) are not represented in the policy at all; requesting them on a target that lacks them always raises. + +### Overriding capabilities per instance + +For targets whose capabilities depend on deployment (HTTP endpoints, Playwright-driven UIs, custom backends), pass a `TargetConfiguration` to the constructor: + +```python +from pyrit.prompt_target.common.target_capabilities import TargetCapabilities +from pyrit.prompt_target.common.target_configuration import TargetConfiguration + +config = TargetConfiguration( + capabilities=TargetCapabilities( + supports_multi_turn=True, + supports_editable_history=True, + supports_system_prompt=True, + ), +) +target = MyHTTPTarget(custom_configuration=config, ...) +``` + +The full implementation lives in [`pyrit/prompt_target/common/target_capabilities.py`](https://github.com/microsoft/PyRIT/blob/main/pyrit/prompt_target/common/target_capabilities.py) and [`pyrit/prompt_target/common/target_configuration.py`](https://github.com/microsoft/PyRIT/blob/main/pyrit/prompt_target/common/target_configuration.py). For runnable examples — inspecting capabilities on a real target, comparing known model profiles, and `ADAPT` vs `RAISE` in action — see [Target Capabilities](./6_1_target_capabilities.ipynb). ## Multi-Modal Targets diff --git a/doc/code/targets/10_3_websocket_copilot_target.ipynb b/doc/code/targets/10_3_websocket_copilot_target.ipynb index 0b4b1c2714..c5b9c77bb6 100644 --- a/doc/code/targets/10_3_websocket_copilot_target.ipynb +++ b/doc/code/targets/10_3_websocket_copilot_target.ipynb @@ -82,7 +82,7 @@ "\n", "The `WebSocketCopilotTarget` supports multi-turn conversations by leveraging Copilot's server-side conversation management. It automatically generates consistent `session_id` and `conversation_id` values for each PyRIT conversation, enabling Copilot to maintain context across multiple turns.\n", "\n", - "However, this target does not support setting a system prompt nor modifying conversation history. As a result, it cannot be used with attack strategies that require altering prior messages (such as PAIR, TAP, or flip attack) or in contexts where a `PromptChatTarget` is required.\n", + " \"However, this target does not support setting a system prompt nor modifying conversation history. As a result, it cannot be used with attack strategies that require altering prior messages (such as PAIR, TAP, or flip attack) or in contexts where a chat-style target (one that declares `supports_multi_turn=True` and `supports_editable_history=True`) is required.\\n\",\n", "\n", "Here is a simple multi-turn conversation example:" ] diff --git a/doc/code/targets/10_3_websocket_copilot_target.py b/doc/code/targets/10_3_websocket_copilot_target.py index 88da2ed729..5279ae8bc7 100644 --- a/doc/code/targets/10_3_websocket_copilot_target.py +++ b/doc/code/targets/10_3_websocket_copilot_target.py @@ -45,7 +45,7 @@ # # The `WebSocketCopilotTarget` supports multi-turn conversations by leveraging Copilot's server-side conversation management. It automatically generates consistent `session_id` and `conversation_id` values for each PyRIT conversation, enabling Copilot to maintain context across multiple turns. # -# However, this target does not support setting a system prompt nor modifying conversation history. As a result, it cannot be used with attack strategies that require altering prior messages (such as PAIR, TAP, or flip attack) or in contexts where a `PromptChatTarget` is required. +# However, this target does not support setting a system prompt nor modifying conversation history. As a result, it cannot be used with attack strategies that require altering prior messages (such as PAIR, TAP, or flip attack) or in contexts where a chat-style target (one that declares `supports_multi_turn=True` and `supports_editable_history=True`) is required. # # Here is a simple multi-turn conversation example: diff --git a/doc/code/targets/6_1_target_capabilities.ipynb b/doc/code/targets/6_1_target_capabilities.ipynb new file mode 100644 index 0000000000..18c1902062 --- /dev/null +++ b/doc/code/targets/6_1_target_capabilities.ipynb @@ -0,0 +1,486 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "0", + "metadata": {}, + "source": [ + "# 6.1 Target Capabilities\n", + "\n", + "Every `PromptTarget` carries a `TargetConfiguration` that declares what it natively supports, what to do\n", + "when a capability is missing, and how to adapt the conversation when adaptation is permitted. This notebook\n", + "walks through how to inspect, validate, and override capabilities on a real target — the same machinery\n", + "attacks, scorers, and converters use under the hood.\n", + "\n", + "A `TargetConfiguration` composes three concerns:\n", + "\n", + "* **`TargetCapabilities`** — declarative, immutable description of what the target natively supports.\n", + "* **`CapabilityHandlingPolicy`** — for each adaptable capability, whether to `ADAPT` (run a normalizer)\n", + " or `RAISE` (fail immediately) when the target lacks it.\n", + "* **`ConversationNormalizationPipeline`** — the ordered set of normalizers derived from the gap between\n", + " the declared capabilities and the policy.\n", + "\n", + "See [Target Capabilities](./0_prompt_targets.md#target-capabilities) in the overview for the full list\n", + "of capability flags." + ] + }, + { + "cell_type": "markdown", + "id": "1", + "metadata": {}, + "source": [ + "## 1. Inspect a real target's configuration\n", + "\n", + "We use `OpenAIChatTarget` throughout this notebook. Constructing the target does not make any network\n", + "calls — we are only inspecting its declared configuration." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Found default environment files: ['./.pyrit/.env']\n", + "Loaded environment file: ./.pyrit/.env\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "No new upgrade operations detected.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "supports_multi_turn: True\n", + "supports_editable_history: True\n", + "supports_system_prompt: True\n", + "supports_json_output: True\n", + "supports_json_schema: False\n", + "input_modalities: [['image_path'], ['image_path', 'text'], ['text']]\n", + "output_modalities: [['text']]\n" + ] + } + ], + "source": [ + "from pyrit.prompt_target import OpenAIChatTarget\n", + "from pyrit.setup import IN_MEMORY, initialize_pyrit_async\n", + "\n", + "await initialize_pyrit_async(memory_db_type=IN_MEMORY) # type: ignore\n", + "\n", + "target = OpenAIChatTarget(model_name=\"gpt-4o\", endpoint=\"https://example.invalid/\", api_key=\"sk-not-a-real-key\")\n", + "caps = target.configuration.capabilities\n", + "\n", + "print(\"supports_multi_turn: \", caps.supports_multi_turn)\n", + "print(\"supports_editable_history: \", caps.supports_editable_history)\n", + "print(\"supports_system_prompt: \", caps.supports_system_prompt)\n", + "print(\"supports_json_output: \", caps.supports_json_output)\n", + "print(\"supports_json_schema: \", caps.supports_json_schema)\n", + "print(\"input_modalities: \", sorted(sorted(m) for m in caps.input_modalities))\n", + "print(\"output_modalities: \", sorted(sorted(m) for m in caps.output_modalities))" + ] + }, + { + "cell_type": "markdown", + "id": "3", + "metadata": {}, + "source": [ + "## 2. Default configurations and known model profiles\n", + "\n", + "Each target class declares a `_DEFAULT_CONFIGURATION` class attribute. For well-known underlying models,\n", + "`get_default_configuration(underlying_model=...)` returns a richer profile from\n", + "`TargetCapabilities.get_known_capabilities` — for example, `gpt-5` gains `supports_json_schema=True`\n", + "and other models pick up the right modality combinations automatically. Unknown models fall back to\n", + "the class default." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "capability class default gpt-4o gpt-5 unknown \n", + "--------------------------------------------------------------------------------\n", + "supports_multi_turn True True True True \n", + "supports_editable_history True True True True \n", + "supports_system_prompt True True True True \n", + "supports_json_output True True True True \n", + "supports_json_schema False False True False \n" + ] + } + ], + "source": [ + "class_default = OpenAIChatTarget._DEFAULT_CONFIGURATION.capabilities\n", + "gpt_4o = OpenAIChatTarget.get_default_configuration(underlying_model=\"gpt-4o\").capabilities\n", + "gpt_5 = OpenAIChatTarget.get_default_configuration(underlying_model=\"gpt-5\").capabilities\n", + "unknown = OpenAIChatTarget.get_default_configuration(underlying_model=\"not-a-real-model\").capabilities\n", + "\n", + "print(f\"{'capability':<32}{'class default':<18}{'gpt-4o':<10}{'gpt-5':<10}{'unknown':<10}\")\n", + "print(\"-\" * 80)\n", + "for flag in (\n", + " \"supports_multi_turn\",\n", + " \"supports_editable_history\",\n", + " \"supports_system_prompt\",\n", + " \"supports_json_output\",\n", + " \"supports_json_schema\",\n", + "):\n", + " row = (\n", + " f\"{flag:<32}\"\n", + " f\"{str(getattr(class_default, flag)):<18}\"\n", + " f\"{str(getattr(gpt_4o, flag)):<10}\"\n", + " f\"{str(getattr(gpt_5, flag)):<10}\"\n", + " f\"{str(getattr(unknown, flag)):<10}\"\n", + " )\n", + " print(row)" + ] + }, + { + "cell_type": "markdown", + "id": "5", + "metadata": {}, + "source": [ + "## 3. Declare and validate consumer requirements\n", + "\n", + "Components that need particular capabilities declare them as a `TargetRequirements` and validate at\n", + "construction time. PyRIT ships a `CHAT_TARGET_REQUIREMENTS` constant for the common case of needing\n", + "multi-turn + editable history — the replacement for the deprecated `PromptChatTarget` type check.\n", + "\n", + "`TargetRequirements.validate` collects every missing capability and raises a single `ValueError` so\n", + "callers see all violations at once." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "OpenAIChatTarget satisfies CHAT_TARGET_REQUIREMENTS\n" + ] + } + ], + "source": [ + "from pyrit.prompt_target import CHAT_TARGET_REQUIREMENTS\n", + "\n", + "CHAT_TARGET_REQUIREMENTS.validate(target=target)\n", + "print(\"OpenAIChatTarget satisfies CHAT_TARGET_REQUIREMENTS\")" + ] + }, + { + "cell_type": "markdown", + "id": "7", + "metadata": {}, + "source": [ + "To check a single capability, call `target.configuration.ensure_can_handle(capability=...)` directly." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Multi-turn check passed\n" + ] + } + ], + "source": [ + "from pyrit.prompt_target.common.target_capabilities import CapabilityName\n", + "\n", + "target.configuration.ensure_can_handle(capability=CapabilityName.MULTI_TURN)\n", + "print(\"Multi-turn check passed\")" + ] + }, + { + "cell_type": "markdown", + "id": "9", + "metadata": {}, + "source": [ + "## 4. Override the configuration per instance\n", + "\n", + "For targets whose capabilities depend on deployment (HTTP endpoints, Playwright UIs, custom backends —\n", + "or simply an OpenAI-compatible model whose actual capabilities differ from `gpt-4o`), pass a\n", + "`TargetConfiguration` via `custom_configuration`. The instance uses your override instead of the class\n", + "default." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "10", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "class default supports_multi_turn: True\n", + "instance supports_multi_turn: False\n", + "\n", + "Validation failed as expected:\n", + "Target does not satisfy 2 required capability(ies):\n", + " - Target does not support 'supports_editable_history' and no handling policy exists for it.\n", + " - Target does not support 'supports_multi_turn' and the handling policy is RAISE.\n" + ] + } + ], + "source": [ + "from pyrit.prompt_target.common.target_capabilities import TargetCapabilities\n", + "from pyrit.prompt_target.common.target_configuration import TargetConfiguration\n", + "\n", + "restricted_config = TargetConfiguration(\n", + " capabilities=TargetCapabilities(\n", + " supports_multi_turn=False,\n", + " supports_system_prompt=False,\n", + " supports_multi_message_pieces=True,\n", + " ),\n", + ")\n", + "restricted_target = OpenAIChatTarget(\n", + " model_name=\"custom-model\",\n", + " endpoint=\"https://example.invalid/\",\n", + " api_key=\"sk-not-a-real-key\",\n", + " custom_configuration=restricted_config,\n", + ")\n", + "\n", + "print(\"class default supports_multi_turn: \", class_default.supports_multi_turn)\n", + "print(\"instance supports_multi_turn: \", restricted_target.configuration.capabilities.supports_multi_turn)\n", + "\n", + "try:\n", + " CHAT_TARGET_REQUIREMENTS.validate(target=restricted_target)\n", + "except ValueError as exc:\n", + " print(\"\\nValidation failed as expected:\")\n", + " print(exc)" + ] + }, + { + "cell_type": "markdown", + "id": "11", + "metadata": {}, + "source": [ + "## 5. ADAPT vs RAISE\n", + "\n", + "When a capability is missing, the `CapabilityHandlingPolicy` decides what happens. Only *adaptable*\n", + "capabilities (currently `MULTI_TURN` and `SYSTEM_PROMPT`) can be papered over by PyRIT — for these,\n", + "you can switch the behavior from `RAISE` (default) to `ADAPT`. With `ADAPT`, the conversation goes\n", + "through a normalizer that flattens history or merges system prompts before reaching the target.\n", + "\n", + "Below we wrap a single-turn endpoint two ways and watch the pipeline change. Note that the `RAISE`\n", + "pipeline is **empty**: when a missing capability is configured to raise, there is nothing to\n", + "normalize. The error surfaces later, when a consumer calls `ensure_can_handle` or\n", + "`TargetRequirements.validate`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "12", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "RAISE pipeline normalizers: []\n", + "ADAPT pipeline normalizers: ['GenericSystemSquashNormalizer', 'HistorySquashNormalizer']\n" + ] + } + ], + "source": [ + "from pyrit.prompt_target.common.target_capabilities import (\n", + " CapabilityHandlingPolicy,\n", + " UnsupportedCapabilityBehavior,\n", + ")\n", + "\n", + "single_turn_caps = TargetCapabilities(supports_multi_turn=False, supports_system_prompt=False)\n", + "\n", + "raise_config = TargetConfiguration(\n", + " capabilities=single_turn_caps,\n", + " policy=CapabilityHandlingPolicy(\n", + " behaviors={\n", + " CapabilityName.MULTI_TURN: UnsupportedCapabilityBehavior.RAISE,\n", + " CapabilityName.SYSTEM_PROMPT: UnsupportedCapabilityBehavior.RAISE,\n", + " }\n", + " ),\n", + ")\n", + "adapt_config = TargetConfiguration(\n", + " capabilities=single_turn_caps,\n", + " policy=CapabilityHandlingPolicy(\n", + " behaviors={\n", + " CapabilityName.MULTI_TURN: UnsupportedCapabilityBehavior.ADAPT,\n", + " CapabilityName.SYSTEM_PROMPT: UnsupportedCapabilityBehavior.ADAPT,\n", + " }\n", + " ),\n", + ")\n", + "\n", + "raise_target = OpenAIChatTarget(\n", + " model_name=\"custom-model\",\n", + " endpoint=\"https://example.invalid/\",\n", + " api_key=\"sk-not-a-real-key\",\n", + " custom_configuration=raise_config,\n", + ")\n", + "adapt_target = OpenAIChatTarget(\n", + " model_name=\"custom-model\",\n", + " endpoint=\"https://example.invalid/\",\n", + " api_key=\"sk-not-a-real-key\",\n", + " custom_configuration=adapt_config,\n", + ")\n", + "\n", + "print(\"RAISE pipeline normalizers: \", [type(n).__name__ for n in raise_target.configuration.pipeline._normalizers])\n", + "print(\"ADAPT pipeline normalizers: \", [type(n).__name__ for n in adapt_target.configuration.pipeline._normalizers])" + ] + }, + { + "cell_type": "markdown", + "id": "13", + "metadata": {}, + "source": [ + "With `ADAPT`, running a multi-turn conversation through `normalize_async` collapses it into a single\n", + "user message — the exact payload the target will see when a prompt is sent." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "14", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "original turns: 3\n", + "normalized turns: 1\n", + "flattened text:\n", + "[Conversation History]\n", + "User: What is the capital of France?\n", + "Assistant: Paris.\n", + "\n", + "[Current Message]\n", + "And of Germany?\n" + ] + } + ], + "source": [ + "from pyrit.models import Message\n", + "\n", + "conversation = [\n", + " Message.from_prompt(prompt=\"What is the capital of France?\", role=\"user\"),\n", + " Message.from_prompt(prompt=\"Paris.\", role=\"assistant\"),\n", + " Message.from_prompt(prompt=\"And of Germany?\", role=\"user\"),\n", + "]\n", + "\n", + "normalized = await adapt_target.configuration.normalize_async(messages=conversation) # type: ignore\n", + "print(f\"original turns: {len(conversation)}\")\n", + "print(f\"normalized turns: {len(normalized)}\")\n", + "print(\"flattened text:\")\n", + "print(normalized[-1].message_pieces[0].original_value)" + ] + }, + { + "cell_type": "markdown", + "id": "15", + "metadata": {}, + "source": [ + "By contrast, the `RAISE` configuration validates eagerly: any consumer requiring `MULTI_TURN` will\n", + "get a `ValueError` before a single prompt is sent." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "16", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Target does not support 'supports_multi_turn' and the handling policy is RAISE.\n" + ] + } + ], + "source": [ + "try:\n", + " raise_target.configuration.ensure_can_handle(capability=CapabilityName.MULTI_TURN)\n", + "except ValueError as exc:\n", + " print(exc)" + ] + }, + { + "cell_type": "markdown", + "id": "17", + "metadata": {}, + "source": [ + "## 6. Non-adaptable capabilities\n", + "\n", + "Some capabilities cannot be safely emulated — for example, `supports_editable_history` is a property\n", + "of the underlying API contract and there is no normalizer that can fake it. These capabilities are\n", + "not represented in the `CapabilityHandlingPolicy` at all; requesting them on a target that lacks\n", + "them always raises, regardless of policy." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "18", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Target does not support 'supports_editable_history' and no handling policy exists for it.\n" + ] + } + ], + "source": [ + "no_editable_history = TargetConfiguration(\n", + " capabilities=TargetCapabilities(supports_multi_turn=True, supports_editable_history=False),\n", + ")\n", + "\n", + "try:\n", + " no_editable_history.ensure_can_handle(capability=CapabilityName.EDITABLE_HISTORY)\n", + "except ValueError as exc:\n", + " print(exc)\n", + "# ---" + ] + } + ], + "metadata": { + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.15" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/doc/code/targets/6_1_target_capabilities.py b/doc/code/targets/6_1_target_capabilities.py new file mode 100644 index 0000000000..985374357b --- /dev/null +++ b/doc/code/targets/6_1_target_capabilities.py @@ -0,0 +1,248 @@ +# --- +# jupyter: +# jupytext: +# text_representation: +# extension: .py +# format_name: percent +# format_version: '1.3' +# jupytext_version: 1.19.0 +# --- + +# %% [markdown] +# # 6.1 Target Capabilities +# +# Every `PromptTarget` carries a `TargetConfiguration` that declares what it natively supports, what to do +# when a capability is missing, and how to adapt the conversation when adaptation is permitted. This notebook +# walks through how to inspect, validate, and override capabilities on a real target — the same machinery +# attacks, scorers, and converters use under the hood. +# +# A `TargetConfiguration` composes three concerns: +# +# * **`TargetCapabilities`** — declarative, immutable description of what the target natively supports. +# * **`CapabilityHandlingPolicy`** — for each adaptable capability, whether to `ADAPT` (run a normalizer) +# or `RAISE` (fail immediately) when the target lacks it. +# * **`ConversationNormalizationPipeline`** — the ordered set of normalizers derived from the gap between +# the declared capabilities and the policy. +# +# See [Target Capabilities](./0_prompt_targets.md#target-capabilities) in the overview for the full list +# of capability flags. + +# %% [markdown] +# ## 1. Inspect a real target's configuration +# +# We use `OpenAIChatTarget` throughout this notebook. Constructing the target does not make any network +# calls — we are only inspecting its declared configuration. + +# %% +from pyrit.prompt_target import OpenAIChatTarget +from pyrit.setup import IN_MEMORY, initialize_pyrit_async + +await initialize_pyrit_async(memory_db_type=IN_MEMORY) # type: ignore + +target = OpenAIChatTarget(model_name="gpt-4o", endpoint="https://example.invalid/", api_key="sk-not-a-real-key") +caps = target.configuration.capabilities + +print("supports_multi_turn: ", caps.supports_multi_turn) +print("supports_editable_history: ", caps.supports_editable_history) +print("supports_system_prompt: ", caps.supports_system_prompt) +print("supports_json_output: ", caps.supports_json_output) +print("supports_json_schema: ", caps.supports_json_schema) +print("input_modalities: ", sorted(sorted(m) for m in caps.input_modalities)) +print("output_modalities: ", sorted(sorted(m) for m in caps.output_modalities)) + +# %% [markdown] +# ## 2. Default configurations and known model profiles +# +# Each target class declares a `_DEFAULT_CONFIGURATION` class attribute. For well-known underlying models, +# `get_default_configuration(underlying_model=...)` returns a richer profile from +# `TargetCapabilities.get_known_capabilities` — for example, `gpt-5` gains `supports_json_schema=True` +# and other models pick up the right modality combinations automatically. Unknown models fall back to +# the class default. + +# %% +class_default = OpenAIChatTarget._DEFAULT_CONFIGURATION.capabilities +gpt_4o = OpenAIChatTarget.get_default_configuration(underlying_model="gpt-4o").capabilities +gpt_5 = OpenAIChatTarget.get_default_configuration(underlying_model="gpt-5").capabilities +unknown = OpenAIChatTarget.get_default_configuration(underlying_model="not-a-real-model").capabilities + +print(f"{'capability':<32}{'class default':<18}{'gpt-4o':<10}{'gpt-5':<10}{'unknown':<10}") +print("-" * 80) +for flag in ( + "supports_multi_turn", + "supports_editable_history", + "supports_system_prompt", + "supports_json_output", + "supports_json_schema", +): + row = ( + f"{flag:<32}" + f"{str(getattr(class_default, flag)):<18}" + f"{str(getattr(gpt_4o, flag)):<10}" + f"{str(getattr(gpt_5, flag)):<10}" + f"{str(getattr(unknown, flag)):<10}" + ) + print(row) + +# %% [markdown] +# ## 3. Declare and validate consumer requirements +# +# Components that need particular capabilities declare them as a `TargetRequirements` and validate at +# construction time. PyRIT ships a `CHAT_TARGET_REQUIREMENTS` constant for the common case of needing +# multi-turn + editable history — the replacement for the deprecated `PromptChatTarget` type check. +# +# `TargetRequirements.validate` collects every missing capability and raises a single `ValueError` so +# callers see all violations at once. + +# %% +from pyrit.prompt_target import CHAT_TARGET_REQUIREMENTS + +CHAT_TARGET_REQUIREMENTS.validate(target=target) +print("OpenAIChatTarget satisfies CHAT_TARGET_REQUIREMENTS") + +# %% [markdown] +# To check a single capability, call `target.configuration.ensure_can_handle(capability=...)` directly. + +# %% +from pyrit.prompt_target.common.target_capabilities import CapabilityName + +target.configuration.ensure_can_handle(capability=CapabilityName.MULTI_TURN) +print("Multi-turn check passed") + +# %% [markdown] +# ## 4. Override the configuration per instance +# +# For targets whose capabilities depend on deployment (HTTP endpoints, Playwright UIs, custom backends — +# or simply an OpenAI-compatible model whose actual capabilities differ from `gpt-4o`), pass a +# `TargetConfiguration` via `custom_configuration`. The instance uses your override instead of the class +# default. + +# %% +from pyrit.prompt_target.common.target_capabilities import TargetCapabilities +from pyrit.prompt_target.common.target_configuration import TargetConfiguration + +restricted_config = TargetConfiguration( + capabilities=TargetCapabilities( + supports_multi_turn=False, + supports_system_prompt=False, + supports_multi_message_pieces=True, + ), +) +restricted_target = OpenAIChatTarget( + model_name="custom-model", + endpoint="https://example.invalid/", + api_key="sk-not-a-real-key", + custom_configuration=restricted_config, +) + +print("class default supports_multi_turn: ", class_default.supports_multi_turn) +print("instance supports_multi_turn: ", restricted_target.configuration.capabilities.supports_multi_turn) + +try: + CHAT_TARGET_REQUIREMENTS.validate(target=restricted_target) +except ValueError as exc: + print("\nValidation failed as expected:") + print(exc) + +# %% [markdown] +# ## 5. ADAPT vs RAISE +# +# When a capability is missing, the `CapabilityHandlingPolicy` decides what happens. Only *adaptable* +# capabilities (currently `MULTI_TURN` and `SYSTEM_PROMPT`) can be papered over by PyRIT — for these, +# you can switch the behavior from `RAISE` (default) to `ADAPT`. With `ADAPT`, the conversation goes +# through a normalizer that flattens history or merges system prompts before reaching the target. +# +# Below we wrap a single-turn endpoint two ways and watch the pipeline change. Note that the `RAISE` +# pipeline is **empty**: when a missing capability is configured to raise, there is nothing to +# normalize. The error surfaces later, when a consumer calls `ensure_can_handle` or +# `TargetRequirements.validate`. + +# %% +from pyrit.prompt_target.common.target_capabilities import ( + CapabilityHandlingPolicy, + UnsupportedCapabilityBehavior, +) + +single_turn_caps = TargetCapabilities(supports_multi_turn=False, supports_system_prompt=False) + +raise_config = TargetConfiguration( + capabilities=single_turn_caps, + policy=CapabilityHandlingPolicy( + behaviors={ + CapabilityName.MULTI_TURN: UnsupportedCapabilityBehavior.RAISE, + CapabilityName.SYSTEM_PROMPT: UnsupportedCapabilityBehavior.RAISE, + } + ), +) +adapt_config = TargetConfiguration( + capabilities=single_turn_caps, + policy=CapabilityHandlingPolicy( + behaviors={ + CapabilityName.MULTI_TURN: UnsupportedCapabilityBehavior.ADAPT, + CapabilityName.SYSTEM_PROMPT: UnsupportedCapabilityBehavior.ADAPT, + } + ), +) + +raise_target = OpenAIChatTarget( + model_name="custom-model", + endpoint="https://example.invalid/", + api_key="sk-not-a-real-key", + custom_configuration=raise_config, +) +adapt_target = OpenAIChatTarget( + model_name="custom-model", + endpoint="https://example.invalid/", + api_key="sk-not-a-real-key", + custom_configuration=adapt_config, +) + +print("RAISE pipeline normalizers: ", [type(n).__name__ for n in raise_target.configuration.pipeline._normalizers]) +print("ADAPT pipeline normalizers: ", [type(n).__name__ for n in adapt_target.configuration.pipeline._normalizers]) + +# %% [markdown] +# With `ADAPT`, running a multi-turn conversation through `normalize_async` collapses it into a single +# user message — the exact payload the target will see when a prompt is sent. + +# %% +from pyrit.models import Message + +conversation = [ + Message.from_prompt(prompt="What is the capital of France?", role="user"), + Message.from_prompt(prompt="Paris.", role="assistant"), + Message.from_prompt(prompt="And of Germany?", role="user"), +] + +normalized = await adapt_target.configuration.normalize_async(messages=conversation) # type: ignore +print(f"original turns: {len(conversation)}") +print(f"normalized turns: {len(normalized)}") +print("flattened text:") +print(normalized[-1].message_pieces[0].original_value) + +# %% [markdown] +# By contrast, the `RAISE` configuration validates eagerly: any consumer requiring `MULTI_TURN` will +# get a `ValueError` before a single prompt is sent. + +# %% +try: + raise_target.configuration.ensure_can_handle(capability=CapabilityName.MULTI_TURN) +except ValueError as exc: + print(exc) + +# %% [markdown] +# ## 6. Non-adaptable capabilities +# +# Some capabilities cannot be safely emulated — for example, `supports_editable_history` is a property +# of the underlying API contract and there is no normalizer that can fake it. These capabilities are +# not represented in the `CapabilityHandlingPolicy` at all; requesting them on a target that lacks +# them always raises, regardless of policy. + +# %% +no_editable_history = TargetConfiguration( + capabilities=TargetCapabilities(supports_multi_turn=True, supports_editable_history=False), +) + +try: + no_editable_history.ensure_can_handle(capability=CapabilityName.EDITABLE_HISTORY) +except ValueError as exc: + print(exc) +# --- diff --git a/doc/code/targets/8_non_llm_targets.ipynb b/doc/code/targets/8_non_llm_targets.ipynb index bc03d603b6..6633d31f2b 100644 --- a/doc/code/targets/8_non_llm_targets.ipynb +++ b/doc/code/targets/8_non_llm_targets.ipynb @@ -10,7 +10,7 @@ "Prompt Targets are most often LLMs, but not always. They should be thought of as anything that you send prompts to.\n", "\n", "\n", - "The `AzureBlobStorageTarget` inherits from `PromptTarget`, meaning it has functionality to send prompts. In contrast to `PromptChatTarget`s, `PromptTarget`s do not interact with chat assistants.\n", + "The `AzureBlobStorageTarget` inherits from `PromptTarget`, meaning it has functionality to send prompts. Unlike chat-style targets (which declare `supports_multi_turn=True` and `supports_editable_history=True`), this target does not interact with chat assistants.\n", "This prompt target in particular will take in a prompt and upload it as a text file to the provided Azure Storage Account Container.\n", "This could be useful for Cross-Prompt Injection Attack scenarios, for example, where there is a jailbreak within a file.\n", "\n", diff --git a/doc/code/targets/8_non_llm_targets.py b/doc/code/targets/8_non_llm_targets.py index fd1eb642b8..3d1356a82c 100644 --- a/doc/code/targets/8_non_llm_targets.py +++ b/doc/code/targets/8_non_llm_targets.py @@ -14,7 +14,7 @@ # Prompt Targets are most often LLMs, but not always. They should be thought of as anything that you send prompts to. # # -# The `AzureBlobStorageTarget` inherits from `PromptTarget`, meaning it has functionality to send prompts. In contrast to `PromptChatTarget`s, `PromptTarget`s do not interact with chat assistants. +# The `AzureBlobStorageTarget` inherits from `PromptTarget`, meaning it has functionality to send prompts. Unlike chat-style targets (which declare `supports_multi_turn=True` and `supports_editable_history=True`), this target does not interact with chat assistants. # This prompt target in particular will take in a prompt and upload it as a text file to the provided Azure Storage Account Container. # This could be useful for Cross-Prompt Injection Attack scenarios, for example, where there is a jailbreak within a file. # diff --git a/doc/myst.yml b/doc/myst.yml index d2decaceca..cbf6114665 100644 --- a/doc/myst.yml +++ b/doc/myst.yml @@ -107,6 +107,7 @@ project: - file: code/targets/4_openai_video_target.ipynb - file: code/targets/5_openai_tts_target.ipynb - file: code/targets/6_custom_targets.ipynb + - file: code/targets/6_1_target_capabilities.ipynb - file: code/targets/7_non_open_ai_chat_targets.ipynb - file: code/targets/8_non_llm_targets.ipynb - file: code/targets/9_rate_limiting.ipynb diff --git a/pyrit/backend/services/converter_service.py b/pyrit/backend/services/converter_service.py index b90887cb4d..64b3d07f30 100644 --- a/pyrit/backend/services/converter_service.py +++ b/pyrit/backend/services/converter_service.py @@ -39,7 +39,7 @@ from pyrit.models import PromptDataType from pyrit.models.data_type_serializer import data_serializer_factory from pyrit.prompt_converter import PromptConverter -from pyrit.prompt_target import PromptChatTarget +from pyrit.prompt_target import PromptTarget from pyrit.registry.object_registries import ConverterRegistry _DATA_TYPE_EXTENSION: dict[str, str] = { @@ -184,7 +184,15 @@ def _extract_parameters(converter_class: type) -> list[ConverterParameterSchema] def _is_llm_based(converter_class: type) -> bool: - """Return True if the converter requires an LLM target parameter.""" + """ + Return True if the converter requires an LLM target parameter. + + A converter is considered LLM-based if any of its ``__init__`` parameters is + annotated as a :class:`PromptTarget` (or subclass). + + Returns: + bool: True if the converter is LLM-based, False otherwise. + """ try: sig = inspect.signature(converter_class.__init__) except (ValueError, TypeError): @@ -197,7 +205,7 @@ def _is_llm_based(converter_class: type) -> bool: if ann is inspect.Parameter.empty: continue try: - if isinstance(ann, type) and issubclass(ann, PromptChatTarget): + if isinstance(ann, type) and issubclass(ann, PromptTarget): return True except TypeError: continue diff --git a/pyrit/executor/attack/component/conversation_manager.py b/pyrit/executor/attack/component/conversation_manager.py index 3089a38109..439f93ac49 100644 --- a/pyrit/executor/attack/component/conversation_manager.py +++ b/pyrit/executor/attack/component/conversation_manager.py @@ -20,8 +20,7 @@ PromptConverterConfiguration, ) from pyrit.prompt_normalizer.prompt_normalizer import PromptNormalizer -from pyrit.prompt_target import PromptTarget -from pyrit.prompt_target.common.target_capabilities import CapabilityName +from pyrit.prompt_target import CapabilityName, PromptTarget if TYPE_CHECKING: from pyrit.executor.attack.core import AttackContext @@ -390,8 +389,8 @@ async def _handle_non_chat_target_async( if config.non_chat_target_behavior == "raise": raise ValueError( - "prepended_conversation requires the objective target to be a chat-capable " - "PromptTarget. Non-chat objective targets do not support conversation history. " + "prepended_conversation requires the objective target to support multi-turn " + "conversations with editable history. The current target does not. " "Use PrependedConversationConfig with non_chat_target_behavior='normalize_first_turn' " "to normalize the conversation into the first message instead." ) diff --git a/pyrit/executor/attack/core/attack_config.py b/pyrit/executor/attack/core/attack_config.py index 7d128ffd79..c86131f769 100644 --- a/pyrit/executor/attack/core/attack_config.py +++ b/pyrit/executor/attack/core/attack_config.py @@ -7,7 +7,7 @@ from pyrit.executor.core import StrategyConverterConfig from pyrit.models import SeedPrompt -from pyrit.prompt_target.common.prompt_chat_target import PromptChatTarget +from pyrit.prompt_target import PromptTarget from pyrit.score import Scorer, TrueFalseScorer @@ -23,7 +23,7 @@ class AttackAdversarialConfig: _DEFAULT_SEED_PROMPT = "" # Adversarial chat target for the attack - target: PromptChatTarget + target: PromptTarget # Path to the YAML file containing the system prompt for the adversarial chat target system_prompt_path: Optional[Union[str, Path]] = None diff --git a/pyrit/executor/attack/core/attack_executor.py b/pyrit/executor/attack/core/attack_executor.py index c5e3816d65..fea9315e0d 100644 --- a/pyrit/executor/attack/core/attack_executor.py +++ b/pyrit/executor/attack/core/attack_executor.py @@ -27,7 +27,7 @@ from pyrit.models import SeedAttackGroup if TYPE_CHECKING: - from pyrit.prompt_target import PromptChatTarget + from pyrit.prompt_target import PromptTarget from pyrit.score import TrueFalseScorer AttackResultT = TypeVar("AttackResultT") @@ -138,7 +138,7 @@ async def execute_attack_from_seed_groups_async( *, attack: AttackStrategy[AttackStrategyContextT, AttackStrategyResultT], seed_groups: Sequence[SeedAttackGroup], - adversarial_chat: Optional["PromptChatTarget"] = None, + adversarial_chat: Optional["PromptTarget"] = None, objective_scorer: Optional["TrueFalseScorer"] = None, field_overrides: Optional[Sequence[dict[str, Any]]] = None, return_partial_on_failure: bool = False, diff --git a/pyrit/executor/attack/core/attack_parameters.py b/pyrit/executor/attack/core/attack_parameters.py index f1fca0f3e5..6dc4166d7e 100644 --- a/pyrit/executor/attack/core/attack_parameters.py +++ b/pyrit/executor/attack/core/attack_parameters.py @@ -10,7 +10,7 @@ from pyrit.models import Message, SeedAttackGroup, SeedGroup if TYPE_CHECKING: - from pyrit.prompt_target import PromptChatTarget + from pyrit.prompt_target import PromptTarget from pyrit.score import TrueFalseScorer AttackParamsT = TypeVar("AttackParamsT", bound="AttackParameters") @@ -78,7 +78,7 @@ async def from_seed_group_async( cls: type[AttackParamsT], *, seed_group: SeedAttackGroup, - adversarial_chat: Optional[PromptChatTarget] = None, + adversarial_chat: Optional[PromptTarget] = None, objective_scorer: Optional[TrueFalseScorer] = None, **overrides: Any, ) -> AttackParamsT: diff --git a/pyrit/executor/attack/multi_turn/crescendo.py b/pyrit/executor/attack/multi_turn/crescendo.py index 4e4fab81ef..4547d5b67a 100644 --- a/pyrit/executor/attack/multi_turn/crescendo.py +++ b/pyrit/executor/attack/multi_turn/crescendo.py @@ -45,8 +45,7 @@ SeedPrompt, ) from pyrit.prompt_normalizer import PromptNormalizer -from pyrit.prompt_target.common.target_capabilities import CapabilityName -from pyrit.prompt_target.common.target_requirements import TargetRequirements +from pyrit.prompt_target import CapabilityName, TargetRequirements from pyrit.score import ( FloatScaleThresholdScorer, Scorer, diff --git a/pyrit/executor/attack/multi_turn/multi_prompt_sending.py b/pyrit/executor/attack/multi_turn/multi_prompt_sending.py index 4d7e8fde02..2f47ece199 100644 --- a/pyrit/executor/attack/multi_turn/multi_prompt_sending.py +++ b/pyrit/executor/attack/multi_turn/multi_prompt_sending.py @@ -28,13 +28,11 @@ SeedAttackGroup, ) from pyrit.prompt_normalizer import PromptNormalizer -from pyrit.prompt_target import PromptTarget -from pyrit.prompt_target.common.target_capabilities import CapabilityName +from pyrit.prompt_target import CapabilityName, PromptTarget from pyrit.prompt_target.common.target_requirements import TargetRequirements from pyrit.score import Scorer if TYPE_CHECKING: - from pyrit.prompt_target import PromptChatTarget from pyrit.score import TrueFalseScorer logger = logging.getLogger(__name__) @@ -56,7 +54,7 @@ async def from_seed_group_async( cls: type["MultiPromptSendingAttackParameters"], seed_group: SeedAttackGroup, *, - adversarial_chat: Optional["PromptChatTarget"] = None, + adversarial_chat: Optional["PromptTarget"] = None, objective_scorer: Optional["TrueFalseScorer"] = None, **overrides: Any, ) -> "MultiPromptSendingAttackParameters": diff --git a/pyrit/executor/attack/multi_turn/multi_turn_attack_strategy.py b/pyrit/executor/attack/multi_turn/multi_turn_attack_strategy.py index 0dba3bb7ad..809b150988 100644 --- a/pyrit/executor/attack/multi_turn/multi_turn_attack_strategy.py +++ b/pyrit/executor/attack/multi_turn/multi_turn_attack_strategy.py @@ -18,7 +18,7 @@ ) from pyrit.memory import CentralMemory from pyrit.models import ConversationReference, ConversationType -from pyrit.prompt_target.common.target_capabilities import CapabilityName +from pyrit.prompt_target import CapabilityName if TYPE_CHECKING: from pyrit.models import ( diff --git a/pyrit/executor/attack/multi_turn/red_teaming.py b/pyrit/executor/attack/multi_turn/red_teaming.py index 878cc8e978..4d5f09b2a2 100644 --- a/pyrit/executor/attack/multi_turn/red_teaming.py +++ b/pyrit/executor/attack/multi_turn/red_teaming.py @@ -38,7 +38,7 @@ SeedPrompt, ) from pyrit.prompt_normalizer import PromptNormalizer -from pyrit.prompt_target.common.target_capabilities import CapabilityName +from pyrit.prompt_target import CapabilityName from pyrit.prompt_target.common.target_requirements import TargetRequirements if TYPE_CHECKING: diff --git a/pyrit/executor/attack/multi_turn/simulated_conversation.py b/pyrit/executor/attack/multi_turn/simulated_conversation.py index c2e52f1600..e25f013f5d 100644 --- a/pyrit/executor/attack/multi_turn/simulated_conversation.py +++ b/pyrit/executor/attack/multi_turn/simulated_conversation.py @@ -26,7 +26,7 @@ if TYPE_CHECKING: from pathlib import Path - from pyrit.prompt_target import PromptChatTarget + from pyrit.prompt_target import PromptTarget from pyrit.score import TrueFalseScorer logger = logging.getLogger(__name__) @@ -35,7 +35,7 @@ async def generate_simulated_conversation_async( *, objective: str, - adversarial_chat: PromptChatTarget, + adversarial_chat: PromptTarget, objective_scorer: TrueFalseScorer, num_turns: int = 3, starting_sequence: int = 0, @@ -170,7 +170,7 @@ async def _generate_next_message_async( *, objective: str, conversation_messages: list[Message], - adversarial_chat: PromptChatTarget, + adversarial_chat: PromptTarget, next_message_system_prompt_path: Union[str, Path], ) -> Message: """ diff --git a/pyrit/executor/attack/multi_turn/tree_of_attacks.py b/pyrit/executor/attack/multi_turn/tree_of_attacks.py index e23af1eabf..886fbbd4cd 100644 --- a/pyrit/executor/attack/multi_turn/tree_of_attacks.py +++ b/pyrit/executor/attack/multi_turn/tree_of_attacks.py @@ -52,8 +52,7 @@ ) from pyrit.models.literals import PromptDataType, PromptResponseError from pyrit.prompt_normalizer import PromptConverterConfiguration, PromptNormalizer -from pyrit.prompt_target import PromptChatTarget, PromptTarget -from pyrit.prompt_target.common.target_capabilities import CapabilityName +from pyrit.prompt_target import CapabilityName, PromptTarget from pyrit.prompt_target.common.target_requirements import TargetRequirements from pyrit.score import ( FloatScaleThresholdScorer, @@ -307,7 +306,7 @@ def __init__( self, *, objective_target: PromptTarget, - adversarial_chat: PromptChatTarget, + adversarial_chat: PromptTarget, adversarial_chat_seed_prompt: SeedPrompt, adversarial_chat_prompt_template: SeedPrompt, adversarial_chat_system_seed_prompt: SeedPrompt, @@ -330,7 +329,7 @@ def __init__( Args: objective_target (PromptTarget): The target to attack. - adversarial_chat (PromptChatTarget): The chat target for generating adversarial prompts. + adversarial_chat (PromptTarget): The chat target for generating adversarial prompts. adversarial_chat_seed_prompt (SeedPrompt): The seed prompt for the first turn. adversarial_chat_prompt_template (SeedPrompt): The template for subsequent turns. adversarial_chat_system_seed_prompt (SeedPrompt): The system prompt for the adversarial chat @@ -1354,7 +1353,7 @@ class TreeOfAttacksWithPruningAttack(AttackStrategy[TAPAttackContext, TAPAttackR def __init__( self, *, - objective_target: PromptTarget = REQUIRED_VALUE, # type: ignore[ty:invalid-assignment, ty:invalid-parameter-default] + objective_target: PromptTarget = REQUIRED_VALUE, # type: ignore[ty:invalid-parameter-default] attack_adversarial_config: AttackAdversarialConfig, attack_converter_config: Optional[AttackConverterConfig] = None, attack_scoring_config: Optional[AttackScoringConfig] = None, @@ -1432,6 +1431,7 @@ def __init__( # Initialize adversarial configuration self._adversarial_chat = attack_adversarial_config.target + # TAP sets a system prompt on the adversarial target and drives a # multi-turn dialogue through it; both capabilities must be native. # (The class-level ``TARGET_REQUIREMENTS`` inherited from ``AttackStrategy`` diff --git a/pyrit/executor/attack/single_turn/context_compliance.py b/pyrit/executor/attack/single_turn/context_compliance.py index 57b9c6d235..61e58b9e5f 100644 --- a/pyrit/executor/attack/single_turn/context_compliance.py +++ b/pyrit/executor/attack/single_turn/context_compliance.py @@ -23,7 +23,7 @@ SeedDataset, ) from pyrit.prompt_normalizer import PromptNormalizer -from pyrit.prompt_target import PromptChatTarget +from pyrit.prompt_target import PromptTarget logger = logging.getLogger(__name__) @@ -57,7 +57,7 @@ class ContextComplianceAttack(PromptSendingAttack): def __init__( self, *, - objective_target: PromptChatTarget = REQUIRED_VALUE, # type: ignore[ty:invalid-parameter-default] + objective_target: PromptTarget = REQUIRED_VALUE, # type: ignore[ty:invalid-parameter-default] attack_adversarial_config: AttackAdversarialConfig, attack_converter_config: Optional[AttackConverterConfig] = None, attack_scoring_config: Optional[AttackScoringConfig] = None, @@ -70,7 +70,7 @@ def __init__( Initialize the context compliance attack strategy. Args: - objective_target (PromptChatTarget): The target system to attack. Must be a PromptChatTarget. + objective_target (PromptTarget): The target system to attack. Must be a PromptTarget. attack_adversarial_config (AttackAdversarialConfig): Configuration for the adversarial component, including the adversarial chat target used for rephrasing. attack_converter_config (Optional[AttackConverterConfig]): Configuration for attack converters, diff --git a/pyrit/executor/attack/single_turn/flip_attack.py b/pyrit/executor/attack/single_turn/flip_attack.py index 322c4c57e8..3497eeec42 100644 --- a/pyrit/executor/attack/single_turn/flip_attack.py +++ b/pyrit/executor/attack/single_turn/flip_attack.py @@ -21,7 +21,7 @@ ) from pyrit.prompt_converter import FlipConverter from pyrit.prompt_normalizer import PromptConverterConfiguration, PromptNormalizer -from pyrit.prompt_target import PromptChatTarget +from pyrit.prompt_target import PromptTarget logger = logging.getLogger(__name__) @@ -40,7 +40,7 @@ class FlipAttack(PromptSendingAttack): def __init__( self, *, - objective_target: PromptChatTarget = REQUIRED_VALUE, # type: ignore[ty:invalid-parameter-default] + objective_target: PromptTarget = REQUIRED_VALUE, # type: ignore[ty:invalid-parameter-default] attack_converter_config: Optional[AttackConverterConfig] = None, attack_scoring_config: Optional[AttackScoringConfig] = None, prompt_normalizer: Optional[PromptNormalizer] = None, @@ -48,7 +48,7 @@ def __init__( ) -> None: """ Args: - objective_target (PromptChatTarget): The target system to attack. + objective_target (PromptTarget): The target system to attack. attack_converter_config (AttackConverterConfig, Optional): Configuration for the prompt converters. attack_scoring_config (AttackScoringConfig, Optional): Configuration for scoring components. prompt_normalizer (PromptNormalizer, Optional): Normalizer for handling prompts. diff --git a/pyrit/executor/promptgen/anecdoctor.py b/pyrit/executor/promptgen/anecdoctor.py index 1d7450f5ce..485690d1e5 100644 --- a/pyrit/executor/promptgen/anecdoctor.py +++ b/pyrit/executor/promptgen/anecdoctor.py @@ -26,7 +26,7 @@ from pyrit.prompt_normalizer import PromptNormalizer if TYPE_CHECKING: - from pyrit.prompt_target import PromptChatTarget + from pyrit.prompt_target import PromptTarget logger = logging.getLogger(__name__) @@ -102,8 +102,8 @@ class AnecdoctorGenerator( def __init__( self, *, - objective_target: PromptChatTarget, - processing_model: Optional[PromptChatTarget] = None, + objective_target: PromptTarget, + processing_model: Optional[PromptTarget] = None, converter_config: Optional[StrategyConverterConfig] = None, prompt_normalizer: Optional[PromptNormalizer] = None, ) -> None: @@ -111,8 +111,8 @@ def __init__( Initialize the Anecdoctor prompt generation strategy. Args: - objective_target (PromptChatTarget): The chat model to be used for prompt generation. - processing_model (Optional[PromptChatTarget]): The model used for knowledge graph extraction. + objective_target (PromptTarget): The chat model to be used for prompt generation. + processing_model (Optional[PromptTarget]): The model used for knowledge graph extraction. If provided, the generator will extract a knowledge graph from the examples before generation. If None, the generator will use few-shot examples directly. converter_config (Optional[StrategyConverterConfig]): Configuration for prompt converters. diff --git a/pyrit/executor/promptgen/fuzzer/fuzzer.py b/pyrit/executor/promptgen/fuzzer/fuzzer.py index 2ebc1e4cd2..0815e5c066 100644 --- a/pyrit/executor/promptgen/fuzzer/fuzzer.py +++ b/pyrit/executor/promptgen/fuzzer/fuzzer.py @@ -36,7 +36,7 @@ if TYPE_CHECKING: from pyrit.executor.promptgen.fuzzer.fuzzer_converter_base import FuzzerConverter - from pyrit.prompt_target import PromptChatTarget, PromptTarget + from pyrit.prompt_target import PromptTarget logger = logging.getLogger(__name__) @@ -539,7 +539,7 @@ def with_default_scorer( *, objective_target: PromptTarget, template_converters: list[FuzzerConverter], - scoring_target: PromptChatTarget, + scoring_target: PromptTarget, converter_config: Optional[StrategyConverterConfig] = None, prompt_normalizer: Optional[PromptNormalizer] = None, frequency_weight: float = _DEFAULT_FREQUENCY_WEIGHT, @@ -562,7 +562,7 @@ def with_default_scorer( Args: objective_target (PromptTarget): The target to send the prompts to. template_converters (List[FuzzerConverter]): The converters to apply on the selected jailbreak template. - scoring_target (PromptChatTarget): The chat target to use for scoring responses. + scoring_target (PromptTarget): The chat target to use for scoring responses. converter_config (Optional[StrategyConverterConfig]): Configuration for prompt converters. prompt_normalizer (Optional[PromptNormalizer]): The prompt normalizer to use. frequency_weight (float): Constant that balances between high reward and selection frequency. diff --git a/pyrit/executor/promptgen/fuzzer/fuzzer_converter_base.py b/pyrit/executor/promptgen/fuzzer/fuzzer_converter_base.py index bba4bfc7c4..cdcc6eaa33 100644 --- a/pyrit/executor/promptgen/fuzzer/fuzzer_converter_base.py +++ b/pyrit/executor/promptgen/fuzzer/fuzzer_converter_base.py @@ -19,7 +19,7 @@ SeedPrompt, ) from pyrit.prompt_converter import ConverterResult, PromptConverter -from pyrit.prompt_target import PromptChatTarget +from pyrit.prompt_target import PromptTarget logger = logging.getLogger(__name__) @@ -40,14 +40,14 @@ class FuzzerConverter(PromptConverter): def __init__( self, *, - converter_target: PromptChatTarget = REQUIRED_VALUE, # type: ignore[ty:invalid-parameter-default] + converter_target: PromptTarget = REQUIRED_VALUE, # type: ignore[ty:invalid-parameter-default] prompt_template: SeedPrompt, ) -> None: """ Initialize the converter with the specified chat target and prompt template. Args: - converter_target (PromptChatTarget): Chat target used to perform fuzzing on user prompt. + converter_target (PromptTarget): Chat target used to perform fuzzing on user prompt. Can be omitted if a default has been configured via PyRIT initialization. prompt_template (SeedPrompt): Template to be used instead of the default system prompt with instructions for the chat target. diff --git a/pyrit/executor/promptgen/fuzzer/fuzzer_crossover_converter.py b/pyrit/executor/promptgen/fuzzer/fuzzer_crossover_converter.py index 6c1b5a0140..be1d4ed24b 100644 --- a/pyrit/executor/promptgen/fuzzer/fuzzer_crossover_converter.py +++ b/pyrit/executor/promptgen/fuzzer/fuzzer_crossover_converter.py @@ -13,7 +13,7 @@ ) from pyrit.models import Message, MessagePiece, PromptDataType, SeedPrompt from pyrit.prompt_converter.prompt_converter import ConverterResult -from pyrit.prompt_target import PromptChatTarget +from pyrit.prompt_target import PromptTarget class FuzzerCrossOverConverter(FuzzerConverter): @@ -25,7 +25,7 @@ class FuzzerCrossOverConverter(FuzzerConverter): def __init__( self, *, - converter_target: Optional[PromptChatTarget] = None, + converter_target: Optional[PromptTarget] = None, prompt_template: Optional[SeedPrompt] = None, prompt_templates: Optional[list[str]] = None, ) -> None: @@ -33,7 +33,7 @@ def __init__( Initialize the converter with the specified chat target and prompt templates. Args: - converter_target (PromptChatTarget): Chat target used to perform fuzzing on user prompt. + converter_target (PromptTarget): Chat target used to perform fuzzing on user prompt. Can be omitted if a default has been configured via PyRIT initialization. prompt_template (SeedPrompt, Optional): Template to be used instead of the default system prompt with instructions for the chat target. diff --git a/pyrit/executor/promptgen/fuzzer/fuzzer_expand_converter.py b/pyrit/executor/promptgen/fuzzer/fuzzer_expand_converter.py index b82ebad0f9..28b1801a72 100644 --- a/pyrit/executor/promptgen/fuzzer/fuzzer_expand_converter.py +++ b/pyrit/executor/promptgen/fuzzer/fuzzer_expand_converter.py @@ -12,7 +12,7 @@ ) from pyrit.models import Message, MessagePiece, PromptDataType, SeedPrompt from pyrit.prompt_converter.prompt_converter import ConverterResult -from pyrit.prompt_target import PromptChatTarget +from pyrit.prompt_target import PromptTarget class FuzzerExpandConverter(FuzzerConverter): @@ -24,7 +24,7 @@ class FuzzerExpandConverter(FuzzerConverter): def __init__( self, *, - converter_target: Optional[PromptChatTarget] = None, + converter_target: Optional[PromptTarget] = None, prompt_template: Optional[SeedPrompt] = None, ) -> None: """Initialize the expand converter with optional chat target and prompt template.""" diff --git a/pyrit/executor/promptgen/fuzzer/fuzzer_rephrase_converter.py b/pyrit/executor/promptgen/fuzzer/fuzzer_rephrase_converter.py index a0a2e2bd1a..10acff3fb6 100644 --- a/pyrit/executor/promptgen/fuzzer/fuzzer_rephrase_converter.py +++ b/pyrit/executor/promptgen/fuzzer/fuzzer_rephrase_converter.py @@ -10,7 +10,7 @@ FuzzerConverter, ) from pyrit.models import SeedPrompt -from pyrit.prompt_target import PromptChatTarget +from pyrit.prompt_target import PromptTarget class FuzzerRephraseConverter(FuzzerConverter): @@ -20,7 +20,7 @@ class FuzzerRephraseConverter(FuzzerConverter): @apply_defaults def __init__( - self, *, converter_target: Optional[PromptChatTarget] = None, prompt_template: Optional[SeedPrompt] = None + self, *, converter_target: Optional[PromptTarget] = None, prompt_template: Optional[SeedPrompt] = None ) -> None: """Initialize the rephrase converter with optional chat target and prompt template.""" prompt_template = ( diff --git a/pyrit/executor/promptgen/fuzzer/fuzzer_shorten_converter.py b/pyrit/executor/promptgen/fuzzer/fuzzer_shorten_converter.py index ca5c8dd8d8..6258a5e7b3 100644 --- a/pyrit/executor/promptgen/fuzzer/fuzzer_shorten_converter.py +++ b/pyrit/executor/promptgen/fuzzer/fuzzer_shorten_converter.py @@ -10,7 +10,7 @@ FuzzerConverter, ) from pyrit.models import SeedPrompt -from pyrit.prompt_target import PromptChatTarget +from pyrit.prompt_target import PromptTarget class FuzzerShortenConverter(FuzzerConverter): @@ -20,7 +20,7 @@ class FuzzerShortenConverter(FuzzerConverter): @apply_defaults def __init__( - self, *, converter_target: Optional[PromptChatTarget] = None, prompt_template: Optional[SeedPrompt] = None + self, *, converter_target: Optional[PromptTarget] = None, prompt_template: Optional[SeedPrompt] = None ) -> None: """Initialize the shorten converter with optional chat target and prompt template.""" prompt_template = ( diff --git a/pyrit/executor/promptgen/fuzzer/fuzzer_similar_converter.py b/pyrit/executor/promptgen/fuzzer/fuzzer_similar_converter.py index 0144cb09b3..d7f2796579 100644 --- a/pyrit/executor/promptgen/fuzzer/fuzzer_similar_converter.py +++ b/pyrit/executor/promptgen/fuzzer/fuzzer_similar_converter.py @@ -10,7 +10,7 @@ FuzzerConverter, ) from pyrit.models import SeedPrompt -from pyrit.prompt_target import PromptChatTarget +from pyrit.prompt_target import PromptTarget class FuzzerSimilarConverter(FuzzerConverter): @@ -20,7 +20,7 @@ class FuzzerSimilarConverter(FuzzerConverter): @apply_defaults def __init__( - self, *, converter_target: Optional[PromptChatTarget] = None, prompt_template: Optional[SeedPrompt] = None + self, *, converter_target: Optional[PromptTarget] = None, prompt_template: Optional[SeedPrompt] = None ) -> None: """Initialize the similar converter with optional chat target and prompt template.""" prompt_template = ( diff --git a/pyrit/memory/azure_sql_memory.py b/pyrit/memory/azure_sql_memory.py index dbb228b435..e04d56f3dd 100644 --- a/pyrit/memory/azure_sql_memory.py +++ b/pyrit/memory/azure_sql_memory.py @@ -846,7 +846,7 @@ def _update_entries(self, *, entries: MutableSequence[Base], update_fields: dict # attributes from the (potentially stale) detached object # and silently overwrite concurrent updates to columns # that are NOT in update_fields. - entry_in_session = session.get(type(entry), entry.id) # type: ignore[ty:unresolved-attribute] + entry_in_session = session.get(type(entry), entry.id) if entry_in_session is None: entry_in_session = session.merge(entry) for field, value in update_fields.items(): diff --git a/pyrit/prompt_converter/llm_generic_text_converter.py b/pyrit/prompt_converter/llm_generic_text_converter.py index 95c4c46a1b..258d94168c 100644 --- a/pyrit/prompt_converter/llm_generic_text_converter.py +++ b/pyrit/prompt_converter/llm_generic_text_converter.py @@ -15,7 +15,7 @@ SeedPrompt, ) from pyrit.prompt_converter.prompt_converter import ConverterResult, PromptConverter -from pyrit.prompt_target import CHAT_CONSUMER_REQUIREMENTS, PromptTarget +from pyrit.prompt_target import CHAT_TARGET_REQUIREMENTS, PromptTarget logger = logging.getLogger(__name__) @@ -27,7 +27,7 @@ class LLMGenericTextConverter(PromptConverter): SUPPORTED_INPUT_TYPES = ("text",) SUPPORTED_OUTPUT_TYPES = ("text",) - TARGET_REQUIREMENTS = CHAT_CONSUMER_REQUIREMENTS + TARGET_REQUIREMENTS = CHAT_TARGET_REQUIREMENTS @apply_defaults def __init__( @@ -43,7 +43,7 @@ def __init__( Args: converter_target (PromptTarget): The endpoint that converts the prompt. Must satisfy - ``CHAT_CONSUMER_REQUIREMENTS`` (multi-turn + editable history capabilities, possibly + ``CHAT_TARGET_REQUIREMENTS`` (multi-turn + editable history capabilities, possibly via normalization-pipeline adaptation). Can be omitted if a default has been configured via PyRIT initialization. system_prompt_template (SeedPrompt, Optional): The prompt template to set as the system prompt. diff --git a/pyrit/prompt_converter/persuasion_converter.py b/pyrit/prompt_converter/persuasion_converter.py index fda8ca8541..cbcfb66d4d 100644 --- a/pyrit/prompt_converter/persuasion_converter.py +++ b/pyrit/prompt_converter/persuasion_converter.py @@ -21,7 +21,7 @@ SeedPrompt, ) from pyrit.prompt_converter.prompt_converter import ConverterResult, PromptConverter -from pyrit.prompt_target import CHAT_CONSUMER_REQUIREMENTS, PromptTarget +from pyrit.prompt_target import CHAT_TARGET_REQUIREMENTS, PromptTarget logger = logging.getLogger(__name__) @@ -47,7 +47,7 @@ class PersuasionConverter(PromptConverter): SUPPORTED_INPUT_TYPES = ("text",) SUPPORTED_OUTPUT_TYPES = ("text",) - TARGET_REQUIREMENTS = CHAT_CONSUMER_REQUIREMENTS + TARGET_REQUIREMENTS = CHAT_TARGET_REQUIREMENTS @apply_defaults def __init__( diff --git a/pyrit/prompt_converter/random_translation_converter.py b/pyrit/prompt_converter/random_translation_converter.py index 4b81d2041b..769cb51611 100644 --- a/pyrit/prompt_converter/random_translation_converter.py +++ b/pyrit/prompt_converter/random_translation_converter.py @@ -35,7 +35,7 @@ class RandomTranslationConverter(LLMGenericTextConverter, WordLevelConverter): def __init__( self, *, - converter_target: PromptTarget = REQUIRED_VALUE, # type: ignore[ty:invalid-assignment, ty:invalid-parameter-default] + converter_target: PromptTarget = REQUIRED_VALUE, # type: ignore[ty:invalid-parameter-default] system_prompt_template: Optional[SeedPrompt] = None, languages: Optional[list[str]] = None, word_selection_strategy: Optional[WordSelectionStrategy] = None, diff --git a/pyrit/prompt_converter/translation_converter.py b/pyrit/prompt_converter/translation_converter.py index 977e92bce5..b940f12c93 100644 --- a/pyrit/prompt_converter/translation_converter.py +++ b/pyrit/prompt_converter/translation_converter.py @@ -24,7 +24,7 @@ SeedPrompt, ) from pyrit.prompt_converter.prompt_converter import ConverterResult, PromptConverter -from pyrit.prompt_target import CHAT_CONSUMER_REQUIREMENTS, PromptTarget +from pyrit.prompt_target import CHAT_TARGET_REQUIREMENTS, PromptTarget logger = logging.getLogger(__name__) @@ -36,7 +36,7 @@ class TranslationConverter(PromptConverter): SUPPORTED_INPUT_TYPES = ("text",) SUPPORTED_OUTPUT_TYPES = ("text",) - TARGET_REQUIREMENTS = CHAT_CONSUMER_REQUIREMENTS + TARGET_REQUIREMENTS = CHAT_TARGET_REQUIREMENTS @apply_defaults def __init__( diff --git a/pyrit/prompt_converter/variation_converter.py b/pyrit/prompt_converter/variation_converter.py index 6924c9207a..44dca26ff0 100644 --- a/pyrit/prompt_converter/variation_converter.py +++ b/pyrit/prompt_converter/variation_converter.py @@ -23,7 +23,7 @@ SeedPrompt, ) from pyrit.prompt_converter.prompt_converter import ConverterResult, PromptConverter -from pyrit.prompt_target import CHAT_CONSUMER_REQUIREMENTS, PromptTarget +from pyrit.prompt_target import CHAT_TARGET_REQUIREMENTS, PromptTarget logger = logging.getLogger(__name__) @@ -35,7 +35,7 @@ class VariationConverter(PromptConverter): SUPPORTED_INPUT_TYPES = ("text",) SUPPORTED_OUTPUT_TYPES = ("text",) - TARGET_REQUIREMENTS = CHAT_CONSUMER_REQUIREMENTS + TARGET_REQUIREMENTS = CHAT_TARGET_REQUIREMENTS @apply_defaults def __init__( diff --git a/pyrit/prompt_target/__init__.py b/pyrit/prompt_target/__init__.py index 3cf51dd68c..489fe34900 100644 --- a/pyrit/prompt_target/__init__.py +++ b/pyrit/prompt_target/__init__.py @@ -23,10 +23,7 @@ UnsupportedCapabilityBehavior, ) from pyrit.prompt_target.common.target_configuration import TargetConfiguration -from pyrit.prompt_target.common.target_requirements import ( - CHAT_CONSUMER_REQUIREMENTS, - TargetRequirements, -) +from pyrit.prompt_target.common.target_requirements import CHAT_TARGET_REQUIREMENTS, TargetRequirements from pyrit.prompt_target.common.utils import limit_requests_per_minute from pyrit.prompt_target.gandalf_target import GandalfLevel, GandalfTarget from pyrit.prompt_target.http_target.http_target import HTTPTarget @@ -75,7 +72,7 @@ def __getattr__(name: str) -> object: "AzureMLChatTarget", "CapabilityName", "CapabilityHandlingPolicy", - "CHAT_CONSUMER_REQUIREMENTS", + "CHAT_TARGET_REQUIREMENTS", "CopilotType", "ConversationNormalizationPipeline", "GandalfLevel", diff --git a/pyrit/prompt_target/azure_ml_chat_target.py b/pyrit/prompt_target/azure_ml_chat_target.py index d4fbf2e534..be4e463779 100644 --- a/pyrit/prompt_target/azure_ml_chat_target.py +++ b/pyrit/prompt_target/azure_ml_chat_target.py @@ -20,7 +20,7 @@ Message, construct_response_from_request, ) -from pyrit.prompt_target.common.prompt_chat_target import PromptChatTarget +from pyrit.prompt_target.common.prompt_target import PromptTarget from pyrit.prompt_target.common.target_capabilities import ( CapabilityHandlingPolicy, CapabilityName, @@ -33,7 +33,7 @@ logger = logging.getLogger(__name__) -class AzureMLChatTarget(PromptChatTarget): +class AzureMLChatTarget(PromptTarget): """ A prompt target for Azure Machine Learning chat endpoints. @@ -149,7 +149,7 @@ def __init__( normalizer_overrides={CapabilityName.SYSTEM_PROMPT: message_normalizer}, ) - PromptChatTarget.__init__( + PromptTarget.__init__( self, max_requests_per_minute=max_requests_per_minute, endpoint=endpoint_value, diff --git a/pyrit/prompt_target/common/prompt_chat_target.py b/pyrit/prompt_target/common/prompt_chat_target.py index c707b7fcad..88ee1c824d 100644 --- a/pyrit/prompt_target/common/prompt_chat_target.py +++ b/pyrit/prompt_target/common/prompt_chat_target.py @@ -1,24 +1,24 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT license. -from typing import Optional +import warnings +from typing import Any -from pyrit.models import MessagePiece -from pyrit.models.json_response_config import _JsonResponseConfig from pyrit.prompt_target.common.prompt_target import PromptTarget -from pyrit.prompt_target.common.target_capabilities import CapabilityName, TargetCapabilities +from pyrit.prompt_target.common.target_capabilities import TargetCapabilities from pyrit.prompt_target.common.target_configuration import TargetConfiguration class PromptChatTarget(PromptTarget): """ - A prompt chat target is a target where you can explicitly set the conversation history using memory. - - Some algorithms require conversation to be modified (e.g. deleting the last message) or set explicitly. - These algorithms will require PromptChatTargets be used. - - As a concrete example, OpenAI chat targets are PromptChatTargets. You can set made-up conversation history. - Realtime chat targets or OpenAI completions are NOT PromptChatTargets. You don't send the conversation history. + .. deprecated:: 0.14.0 + ``PromptChatTarget`` is deprecated and will be removed in v0.16.0. Use + :class:`PromptTarget` directly with a ``TargetConfiguration`` declaring + ``supports_multi_turn=True`` and ``supports_editable_history=True``. + + Backwards-compatible alias for :class:`PromptTarget`. All chat-target functionality + (``set_system_prompt``, ``is_response_format_json``) lives on :class:`PromptTarget`. + Subclassing or instantiating this class emits a :class:`DeprecationWarning`. """ _DEFAULT_CONFIGURATION: TargetConfiguration = TargetConfiguration( @@ -30,75 +30,31 @@ class PromptChatTarget(PromptTarget): ) ) - def __init__( - self, - *, - max_requests_per_minute: Optional[int] = None, - endpoint: str = "", - model_name: str = "", - underlying_model: Optional[str] = None, - custom_configuration: Optional[TargetConfiguration] = None, - custom_capabilities: Optional[TargetCapabilities] = None, - ) -> None: + def __init_subclass__(cls, **kwargs: Any) -> None: """ - Initialize the PromptChatTarget. - - Args: - max_requests_per_minute (int, Optional): Maximum number of requests per minute. - endpoint (str): The endpoint URL. Defaults to empty string. - model_name (str): The model name. Defaults to empty string. - underlying_model (str, Optional): The underlying model name (e.g., "gpt-4o") for - identification purposes. This is useful when the deployment name in Azure differs - from the actual model. Defaults to None. - custom_configuration (TargetConfiguration, Optional): Override the default configuration - for this target instance. If None, uses the class-level defaults. Defaults to None. - custom_capabilities (TargetCapabilities, Optional): **Deprecated.** Use - ``custom_configuration`` instead. Will be removed in v0.14.0. + Call the superclass __init_subclass__ and emit a deprecation warning when subclassing PromptChatTarget. + Use PromptTarget with an appropriate TargetConfiguration instead. """ - super().__init__( - max_requests_per_minute=max_requests_per_minute, - endpoint=endpoint, - model_name=model_name, - underlying_model=underlying_model, - custom_configuration=custom_configuration, - custom_capabilities=custom_capabilities, + super().__init_subclass__(**kwargs) + warnings.warn( + f"Subclassing PromptChatTarget is deprecated and will be removed in v0.16.0. " + f"Inherit from PromptTarget directly and declare supports_multi_turn=True and " + f"supports_editable_history=True in your _DEFAULT_CONFIGURATION. " + f"({cls.__name__})", + DeprecationWarning, + stacklevel=2, ) - def is_response_format_json(self, message_piece: MessagePiece) -> bool: - """ - Check if the response format is JSON and ensure the target supports it. - - Args: - message_piece: A MessagePiece object with a `prompt_metadata` dictionary that may - include a "response_format" key. - - Returns: - bool: True if the response format is JSON, False otherwise. - - Raises: - ValueError: If "json" response format is requested but unsupported. + def __init__(self, *args: Any, **kwargs: Any) -> None: """ - config = self._get_json_response_config(message_piece=message_piece) - return config.enabled - - def _get_json_response_config(self, *, message_piece: MessagePiece) -> _JsonResponseConfig: + Initialize the PromptChatTarget. This constructor is deprecated and will emit a warning. + Use PromptTarget with an appropriate TargetConfiguration instead. """ - Get the JSON response configuration from the message piece metadata. - - Args: - message_piece: A MessagePiece object with a `prompt_metadata` dictionary that may - include JSON response configuration. - - Returns: - _JsonResponseConfig: The JSON response configuration. - - Raises: - ValueError: If JSON response format is requested but unsupported. - """ - config = _JsonResponseConfig.from_metadata(metadata=message_piece.prompt_metadata) - - if config.enabled and not self.configuration.includes(capability=CapabilityName.JSON_OUTPUT): - target_name = self.get_identifier().class_name - raise ValueError(f"This target {target_name} does not support JSON response format.") - - return config + warnings.warn( + "PromptChatTarget is deprecated and will be removed in v0.16.0. " + "Use PromptTarget directly with a TargetConfiguration declaring " + "supports_multi_turn=True and supports_editable_history=True.", + DeprecationWarning, + stacklevel=2, + ) + super().__init__(*args, **kwargs) diff --git a/pyrit/prompt_target/common/prompt_target.py b/pyrit/prompt_target/common/prompt_target.py index 9f115413ba..a4cf2a8a96 100644 --- a/pyrit/prompt_target/common/prompt_target.py +++ b/pyrit/prompt_target/common/prompt_target.py @@ -10,6 +10,7 @@ from pyrit.identifiers import ComponentIdentifier, Identifiable from pyrit.memory import CentralMemory, MemoryInterface from pyrit.models import Message, MessagePiece +from pyrit.models.json_response_config import _JsonResponseConfig from pyrit.prompt_target.common.target_capabilities import CapabilityName, TargetCapabilities from pyrit.prompt_target.common.target_configuration import TargetConfiguration, resolve_configuration_compat @@ -302,6 +303,7 @@ def set_system_prompt( labels (dict[str, str] | None): Optional labels. Raises: + ValueError: If the target does not support multi-turn or editable history. RuntimeError: If the conversation already has messages. """ if labels is not None: @@ -311,6 +313,12 @@ def set_system_prompt( removed_in="0.16.0", ) + if not self.capabilities.supports_multi_turn or not self.capabilities.supports_editable_history: + raise ValueError( + f"Target {type(self).__name__} does not support setting a system prompt. " + "It must support both multi-turn conversations and editable history." + ) + messages = self._memory.get_conversation(conversation_id=conversation_id) if messages: @@ -456,3 +464,42 @@ def _build_identifier(self) -> ComponentIdentifier: ComponentIdentifier: The identifier for this prompt target. """ return self._create_identifier() + + def is_response_format_json(self, message_piece: MessagePiece) -> bool: + """ + Check if the response format is JSON and ensure the target supports it. + + Args: + message_piece: A MessagePiece object with a `prompt_metadata` dictionary that may + include a "response_format" key. + + Returns: + bool: True if the response format is JSON, False otherwise. + + Raises: + ValueError: If "json" response format is requested but unsupported. + """ + config = self._get_json_response_config(message_piece=message_piece) + return config.enabled + + def _get_json_response_config(self, *, message_piece: MessagePiece) -> _JsonResponseConfig: + """ + Get the JSON response configuration from the message piece metadata. + + Args: + message_piece: A MessagePiece object with a `prompt_metadata` dictionary that may + include JSON response configuration. + + Returns: + _JsonResponseConfig: The JSON response configuration. + + Raises: + ValueError: If JSON response format is requested but unsupported. + """ + config = _JsonResponseConfig.from_metadata(metadata=message_piece.prompt_metadata) + + if config.enabled and not self.capabilities.supports_json_output: + target_name = self.get_identifier().class_name + raise ValueError(f"This target {target_name} does not support JSON response format.") + + return config diff --git a/pyrit/prompt_target/common/target_requirements.py b/pyrit/prompt_target/common/target_requirements.py index 57c96312e9..f89c8b853e 100644 --- a/pyrit/prompt_target/common/target_requirements.py +++ b/pyrit/prompt_target/common/target_requirements.py @@ -73,9 +73,20 @@ def validate(self, *, target: PromptTarget) -> None: ) -# Shared requirement used by scorers and converters that set a system prompt -# and drive a short multi-turn conversation. Adaptation is acceptable, native -# support is not required. -CHAT_CONSUMER_REQUIREMENTS = TargetRequirements( - required=frozenset({CapabilityName.EDITABLE_HISTORY, CapabilityName.MULTI_TURN}), -) +def _build_chat_target_requirements() -> TargetRequirements: + """ + Build the requirements for a chat-style target (multi-turn with editable history). + + Returns: + TargetRequirements: The requirements for a chat-style target. + """ + return TargetRequirements(required=frozenset({CapabilityName.MULTI_TURN, CapabilityName.EDITABLE_HISTORY})) + + +CHAT_TARGET_REQUIREMENTS: TargetRequirements = _build_chat_target_requirements() +""" +Standard requirements for a chat-style target: must support multi-turn conversations +with an editable history. This is the replacement for the deprecated +``PromptChatTarget`` type-based check; consumers validate their target against +these requirements at construction time. +""" diff --git a/pyrit/prompt_target/common/utils.py b/pyrit/prompt_target/common/utils.py index ca0a4ca7da..9204a52d57 100644 --- a/pyrit/prompt_target/common/utils.py +++ b/pyrit/prompt_target/common/utils.py @@ -39,7 +39,7 @@ def validate_top_p(top_p: Optional[float]) -> None: def limit_requests_per_minute(func: Callable[..., Any]) -> Callable[..., Any]: """ Enforce rate limit of the target through setting requests per minute. - This should be applied to all send_prompt_async() functions on PromptTarget and PromptChatTarget. + This should be applied to all send_prompt_async() functions on PromptTarget. Args: func (Callable): The function to be decorated. diff --git a/pyrit/prompt_target/hugging_face/hugging_face_chat_target.py b/pyrit/prompt_target/hugging_face/hugging_face_chat_target.py index 193429d021..472ff0c807 100644 --- a/pyrit/prompt_target/hugging_face/hugging_face_chat_target.py +++ b/pyrit/prompt_target/hugging_face/hugging_face_chat_target.py @@ -21,7 +21,7 @@ from pyrit.exceptions import EmptyResponseException, pyrit_target_retry from pyrit.identifiers import ComponentIdentifier from pyrit.models import Message, construct_response_from_request -from pyrit.prompt_target.common.prompt_chat_target import PromptChatTarget +from pyrit.prompt_target.common.prompt_target import PromptTarget from pyrit.prompt_target.common.target_capabilities import TargetCapabilities from pyrit.prompt_target.common.target_configuration import TargetConfiguration from pyrit.prompt_target.common.utils import limit_requests_per_minute @@ -29,7 +29,7 @@ logger = logging.getLogger(__name__) -class HuggingFaceChatTarget(PromptChatTarget): +class HuggingFaceChatTarget(PromptTarget): """ The HuggingFaceChatTarget interacts with HuggingFace models, specifically for conducting red teaming activities. Inherits from PromptTarget to comply with the current design standards. diff --git a/pyrit/prompt_target/openai/openai_chat_target.py b/pyrit/prompt_target/openai/openai_chat_target.py index 6dfb5f391f..3a64471794 100644 --- a/pyrit/prompt_target/openai/openai_chat_target.py +++ b/pyrit/prompt_target/openai/openai_chat_target.py @@ -24,7 +24,7 @@ data_serializer_factory, ) from pyrit.models.json_response_config import _JsonResponseConfig -from pyrit.prompt_target.common.prompt_chat_target import PromptChatTarget +from pyrit.prompt_target.common.prompt_target import PromptTarget from pyrit.prompt_target.common.target_capabilities import TargetCapabilities from pyrit.prompt_target.common.target_configuration import TargetConfiguration, resolve_configuration_compat from pyrit.prompt_target.common.utils import limit_requests_per_minute, validate_temperature, validate_top_p @@ -34,7 +34,7 @@ logger = logging.getLogger(__name__) -class OpenAIChatTarget(OpenAITarget, PromptChatTarget): +class OpenAIChatTarget(OpenAITarget, PromptTarget): """ Facilitates multimodal (image and text) input and text output generation. @@ -69,10 +69,10 @@ class OpenAIChatTarget(OpenAITarget, PromptChatTarget): _DEFAULT_CONFIGURATION: TargetConfiguration = TargetConfiguration( capabilities=TargetCapabilities( supports_multi_turn=True, + supports_editable_history=True, supports_json_output=True, supports_multi_message_pieces=True, supports_system_prompt=True, - supports_editable_history=True, input_modalities=frozenset( {frozenset({"text"}), frozenset({"image_path"}), frozenset({"text", "image_path"})} ), diff --git a/pyrit/prompt_target/openai/openai_realtime_target.py b/pyrit/prompt_target/openai/openai_realtime_target.py index 6dd34c39b1..2674150b1c 100644 --- a/pyrit/prompt_target/openai/openai_realtime_target.py +++ b/pyrit/prompt_target/openai/openai_realtime_target.py @@ -21,7 +21,7 @@ construct_response_from_request, data_serializer_factory, ) -from pyrit.prompt_target.common.prompt_chat_target import PromptChatTarget +from pyrit.prompt_target.common.prompt_target import PromptTarget from pyrit.prompt_target.common.target_capabilities import TargetCapabilities from pyrit.prompt_target.common.target_configuration import TargetConfiguration from pyrit.prompt_target.common.utils import limit_requests_per_minute @@ -58,7 +58,7 @@ def flatten_transcripts(self) -> str: return "".join(self.transcripts) -class RealtimeTarget(OpenAITarget, PromptChatTarget): +class RealtimeTarget(OpenAITarget, PromptTarget): """ A prompt target for Azure OpenAI Realtime API. @@ -72,6 +72,7 @@ class RealtimeTarget(OpenAITarget, PromptChatTarget): _DEFAULT_CONFIGURATION: TargetConfiguration = TargetConfiguration( capabilities=TargetCapabilities( supports_multi_turn=True, + supports_editable_history=True, supports_multi_message_pieces=True, supports_system_prompt=True, input_modalities=frozenset( diff --git a/pyrit/prompt_target/openai/openai_response_target.py b/pyrit/prompt_target/openai/openai_response_target.py index dbe71e5406..8ed04fe79b 100644 --- a/pyrit/prompt_target/openai/openai_response_target.py +++ b/pyrit/prompt_target/openai/openai_response_target.py @@ -28,7 +28,7 @@ PromptResponseError, ) from pyrit.models.json_response_config import _JsonResponseConfig -from pyrit.prompt_target.common.prompt_chat_target import PromptChatTarget +from pyrit.prompt_target.common.prompt_target import PromptTarget from pyrit.prompt_target.common.target_capabilities import TargetCapabilities from pyrit.prompt_target.common.target_configuration import TargetConfiguration from pyrit.prompt_target.common.utils import limit_requests_per_minute, validate_temperature, validate_top_p @@ -59,7 +59,7 @@ class MessagePieceType(str, Enum): MCP_APPROVAL_REQUEST = "mcp_approval_request" -class OpenAIResponseTarget(OpenAITarget, PromptChatTarget): +class OpenAIResponseTarget(OpenAITarget, PromptTarget): """ Enables communication with endpoints that support the OpenAI Response API. @@ -72,6 +72,7 @@ class OpenAIResponseTarget(OpenAITarget, PromptChatTarget): _DEFAULT_CONFIGURATION: TargetConfiguration = TargetConfiguration( capabilities=TargetCapabilities( supports_multi_turn=True, + supports_editable_history=True, supports_json_output=True, supports_multi_message_pieces=True, supports_system_prompt=True, diff --git a/pyrit/registry/object_registries/attack_technique_registry.py b/pyrit/registry/object_registries/attack_technique_registry.py index fb541e5c42..1c5812df06 100644 --- a/pyrit/registry/object_registries/attack_technique_registry.py +++ b/pyrit/registry/object_registries/attack_technique_registry.py @@ -27,7 +27,7 @@ AttackScoringConfig, ) from pyrit.models import SeedAttackTechniqueGroup - from pyrit.prompt_target import PromptChatTarget, PromptTarget + from pyrit.prompt_target import PromptTarget from pyrit.registry.tag_query import TagQuery from pyrit.scenario.core.attack_technique import AttackTechnique from pyrit.scenario.core.attack_technique_factory import AttackTechniqueFactory @@ -92,7 +92,7 @@ class AttackTechniqueSpec: name: str attack_class: type strategy_tags: list[str] = field(default_factory=list) - adversarial_chat: PromptChatTarget | None = field(default=None) + adversarial_chat: PromptTarget | None = field(default=None) adversarial_chat_key: str | None = None extra_kwargs: dict[str, Any] = field(default_factory=dict) accepts_scorer_override: bool = True diff --git a/pyrit/scenario/core/atomic_attack.py b/pyrit/scenario/core/atomic_attack.py index dcdf106883..d63c5e65f5 100644 --- a/pyrit/scenario/core/atomic_attack.py +++ b/pyrit/scenario/core/atomic_attack.py @@ -27,7 +27,7 @@ from pyrit.scenario.core.attack_technique import AttackTechnique if TYPE_CHECKING: - from pyrit.prompt_target import PromptChatTarget + from pyrit.prompt_target import PromptTarget from pyrit.score import TrueFalseScorer logger = logging.getLogger(__name__) @@ -57,7 +57,7 @@ def __init__( attack_technique: AttackTechnique | None = None, attack: AttackStrategy[Any, Any] | None = None, seed_groups: list[SeedAttackGroup], - adversarial_chat: Optional["PromptChatTarget"] = None, + adversarial_chat: Optional["PromptTarget"] = None, objective_scorer: Optional["TrueFalseScorer"] = None, memory_labels: Optional[dict[str, str]] = None, **attack_execute_params: Any, diff --git a/pyrit/scenario/core/attack_technique_factory.py b/pyrit/scenario/core/attack_technique_factory.py index 1d9bdbed4a..f636e829c4 100644 --- a/pyrit/scenario/core/attack_technique_factory.py +++ b/pyrit/scenario/core/attack_technique_factory.py @@ -25,7 +25,7 @@ AttackScoringConfig, ) from pyrit.models import SeedAttackTechniqueGroup - from pyrit.prompt_target import PromptChatTarget, PromptTarget + from pyrit.prompt_target import PromptTarget class AttackTechniqueFactory(Identifiable): @@ -138,7 +138,7 @@ def seed_technique(self) -> SeedAttackTechniqueGroup | None: return self._seed_technique @property - def adversarial_chat(self) -> PromptChatTarget | None: + def adversarial_chat(self) -> PromptTarget | None: """The adversarial chat target baked into this factory, or None.""" return self._adversarial_config.target if self._adversarial_config else None diff --git a/pyrit/scenario/core/scenario_techniques.py b/pyrit/scenario/core/scenario_techniques.py index 818ba8a530..117bad082c 100644 --- a/pyrit/scenario/core/scenario_techniques.py +++ b/pyrit/scenario/core/scenario_techniques.py @@ -34,8 +34,7 @@ ) from pyrit.models import SeedAttackTechniqueGroup, SeedSimulatedConversation from pyrit.models.seeds.seed_simulated_conversation import NextMessageSystemPromptPaths -from pyrit.prompt_target import OpenAIChatTarget, PromptChatTarget -from pyrit.prompt_target.common.target_capabilities import CapabilityName +from pyrit.prompt_target import CapabilityName, OpenAIChatTarget, PromptTarget from pyrit.registry import TargetRegistry from pyrit.registry.object_registries.attack_technique_registry import ( AttackTechniqueRegistry, @@ -104,7 +103,7 @@ # --------------------------------------------------------------------------- -def get_default_adversarial_target() -> PromptChatTarget: +def get_default_adversarial_target() -> PromptTarget: """ Resolve the default adversarial chat target. @@ -114,7 +113,7 @@ def get_default_adversarial_target() -> PromptChatTarget: ``@apply_defaults`` resolution. Returns: - PromptChatTarget: The resolved adversarial chat target. + PromptTarget: The resolved adversarial chat target. Raises: ValueError: If the registered target does not support multi-turn. @@ -161,7 +160,7 @@ def build_scenario_techniques() -> list[AttackTechniqueSpec]: ValueError: If a spec declares ``adversarial_chat_key`` but the key is not found in ``TargetRegistry``. """ - default_adversarial: PromptChatTarget | None = None + default_adversarial: PromptTarget | None = None result = [] for spec in SCENARIO_TECHNIQUES: diff --git a/pyrit/scenario/scenarios/airt/leakage.py b/pyrit/scenario/scenarios/airt/leakage.py index 801522d82e..df80784bf6 100644 --- a/pyrit/scenario/scenarios/airt/leakage.py +++ b/pyrit/scenario/scenarios/airt/leakage.py @@ -22,7 +22,7 @@ from pyrit.models import SeedAttackGroup, SeedObjective from pyrit.prompt_converter import AddImageTextConverter, FirstLetterConverter, PromptConverter from pyrit.prompt_normalizer import PromptConverterConfiguration -from pyrit.prompt_target import OpenAIChatTarget, PromptChatTarget +from pyrit.prompt_target import OpenAIChatTarget, PromptTarget from pyrit.scenario.core.atomic_attack import AtomicAttack from pyrit.scenario.core.attack_technique import AttackTechnique from pyrit.scenario.core.dataset_configuration import DatasetConfiguration @@ -129,7 +129,7 @@ def default_dataset_config(cls) -> DatasetConfiguration: def __init__( self, *, - adversarial_chat: Optional[PromptChatTarget] = None, + adversarial_chat: Optional[PromptTarget] = None, objectives: Optional[list[str]] = None, objective_scorer: Optional[TrueFalseScorer] = None, include_baseline: bool = True, @@ -139,7 +139,7 @@ def __init__( Initialize the leakage scenario. Args: - adversarial_chat (Optional[PromptChatTarget]): Adversarial chat target for multi-turn attacks + adversarial_chat (Optional[PromptTarget]): Adversarial chat target for multi-turn attacks (Crescendo, RolePlay). If not provided, defaults to an OpenAI chat target. objectives (Optional[List[str]]): List of objectives to test for data leakage. If not provided, defaults to objectives from the airt_leakage dataset. diff --git a/pyrit/scenario/scenarios/airt/psychosocial.py b/pyrit/scenario/scenarios/airt/psychosocial.py index 8a0fc924b9..ff98853747 100644 --- a/pyrit/scenario/scenarios/airt/psychosocial.py +++ b/pyrit/scenario/scenarios/airt/psychosocial.py @@ -27,9 +27,8 @@ from pyrit.prompt_normalizer.prompt_converter_configuration import ( PromptConverterConfiguration, ) -from pyrit.prompt_target import OpenAIChatTarget, PromptChatTarget -from pyrit.prompt_target.common.target_capabilities import CapabilityName -from pyrit.prompt_target.common.target_requirements import TargetRequirements +from pyrit.prompt_target import CapabilityName, OpenAIChatTarget, PromptTarget +from pyrit.prompt_target.common.target_requirements import CHAT_TARGET_REQUIREMENTS, TargetRequirements from pyrit.scenario.core.atomic_attack import AtomicAttack from pyrit.scenario.core.attack_technique import AttackTechnique from pyrit.scenario.core.dataset_configuration import DatasetConfiguration @@ -211,7 +210,7 @@ def __init__( self, *, objectives: Optional[list[str]] = None, - adversarial_chat: Optional[PromptChatTarget] = None, + adversarial_chat: Optional[PromptTarget] = None, objective_scorer: Optional[FloatScaleThresholdScorer] = None, scenario_result_id: Optional[str] = None, subharm_configs: Optional[dict[str, SubharmConfig]] = None, @@ -223,7 +222,7 @@ def __init__( Args: objectives (Optional[List[str]]): DEPRECATED - Use dataset_config in initialize_async instead. List of objectives to test for psychosocial harms. - adversarial_chat (Optional[PromptChatTarget]): Additionally used for adversarial attacks + adversarial_chat (Optional[PromptTarget]): Additionally used for adversarial attacks and scoring defaults. If not provided, a default OpenAI target will be created using environment variables. objective_scorer (Optional[FloatScaleThresholdScorer]): Scorer to evaluate attack success. @@ -430,6 +429,14 @@ def _get_scorer(self, subharm: Optional[str] = None) -> FloatScaleThresholdScore async def _get_atomic_attacks_async(self) -> list[AtomicAttack]: if self._objective_target is None: raise ValueError("objective_target must be set before creating attacks") + try: + CHAT_TARGET_REQUIREMENTS.validate(target=self._objective_target) + except ValueError as exc: + raise TypeError( + f"PsychosocialHarmsScenario requires a target that supports multi-turn " + f"conversations with editable history. Target {type(self._objective_target).__name__} " + f"does not satisfy these requirements: {exc}" + ) from exc resolved = self._resolve_seed_groups() self._seed_groups = resolved.seed_groups diff --git a/pyrit/scenario/scenarios/airt/scam.py b/pyrit/scenario/scenarios/airt/scam.py index e714b73a57..789a3eaf68 100644 --- a/pyrit/scenario/scenarios/airt/scam.py +++ b/pyrit/scenario/scenarios/airt/scam.py @@ -23,7 +23,7 @@ AttackScoringConfig, ) from pyrit.models import SeedAttackGroup -from pyrit.prompt_target import OpenAIChatTarget, PromptChatTarget +from pyrit.prompt_target import OpenAIChatTarget, PromptTarget from pyrit.scenario.core.atomic_attack import AtomicAttack from pyrit.scenario.core.attack_technique import AttackTechnique from pyrit.scenario.core.dataset_configuration import DatasetConfiguration @@ -149,7 +149,7 @@ def __init__( self, *, objective_scorer: Optional[TrueFalseScorer] = None, - adversarial_chat: Optional[PromptChatTarget] = None, + adversarial_chat: Optional[PromptTarget] = None, include_baseline: bool = True, scenario_result_id: Optional[str] = None, ) -> None: @@ -159,7 +159,7 @@ def __init__( Args: objective_scorer (Optional[TrueFalseScorer]): Custom scorer for objective evaluation. - adversarial_chat (Optional[PromptChatTarget]): Chat target used to rephrase the + adversarial_chat (Optional[PromptTarget]): Chat target used to rephrase the objective into the role-play context (in single-turn strategies). include_baseline (bool): Whether to include a baseline atomic attack that sends all objectives without modifications. Defaults to True. When True, a "baseline" attack is automatically diff --git a/pyrit/scenario/scenarios/foundry/red_team_agent.py b/pyrit/scenario/scenarios/foundry/red_team_agent.py index a875e186a5..8b7da826e6 100644 --- a/pyrit/scenario/scenarios/foundry/red_team_agent.py +++ b/pyrit/scenario/scenarios/foundry/red_team_agent.py @@ -61,7 +61,6 @@ PromptConverterConfiguration, ) from pyrit.prompt_target import PromptTarget -from pyrit.prompt_target.common.prompt_chat_target import PromptChatTarget from pyrit.prompt_target.openai.openai_chat_target import OpenAIChatTarget from pyrit.scenario.core.atomic_attack import AtomicAttack from pyrit.scenario.core.attack_technique import AttackTechnique @@ -246,7 +245,7 @@ def default_dataset_config(cls) -> DatasetConfiguration: def __init__( self, *, - adversarial_chat: Optional[PromptChatTarget] = None, + adversarial_chat: Optional[PromptTarget] = None, attack_scoring_config: Optional[AttackScoringConfig] = None, include_baseline: bool = True, scenario_result_id: Optional[str] = None, @@ -255,7 +254,7 @@ def __init__( Initialize a Foundry Scenario with the specified attack strategies. Args: - adversarial_chat (Optional[PromptChatTarget]): Target for multi-turn attacks + adversarial_chat (Optional[PromptTarget]): Target for multi-turn attacks like Crescendo and RedTeaming. Additionally used for scoring defaults. If not provided, a default OpenAI target will be created using environment variables. attack_scoring_config (Optional[AttackScoringConfig]): Configuration for attack scoring, diff --git a/pyrit/score/float_scale/azure_content_filter_scorer.py b/pyrit/score/float_scale/azure_content_filter_scorer.py index 754a0269ce..34168b41ae 100644 --- a/pyrit/score/float_scale/azure_content_filter_scorer.py +++ b/pyrit/score/float_scale/azure_content_filter_scorer.py @@ -149,7 +149,7 @@ def __init__( if callable(self._api_key): # Token provider - create an AsyncTokenCredential wrapper credential = AsyncTokenProviderCredential(self._api_key) # type: ignore[ty:invalid-argument-type] - self._azure_cf_client = ContentSafetyClient(self._endpoint, credential=credential) # type: ignore[ty:invalid-argument-type] + self._azure_cf_client = ContentSafetyClient(self._endpoint, credential=credential) else: # String API key if not isinstance(self._api_key, str): diff --git a/pyrit/score/float_scale/insecure_code_scorer.py b/pyrit/score/float_scale/insecure_code_scorer.py index b8986213e9..9013e584e9 100644 --- a/pyrit/score/float_scale/insecure_code_scorer.py +++ b/pyrit/score/float_scale/insecure_code_scorer.py @@ -9,7 +9,7 @@ from pyrit.exceptions.exception_classes import InvalidJsonException from pyrit.identifiers import ComponentIdentifier from pyrit.models import MessagePiece, Score, SeedPrompt -from pyrit.prompt_target import CHAT_CONSUMER_REQUIREMENTS, PromptTarget +from pyrit.prompt_target import CHAT_TARGET_REQUIREMENTS, PromptTarget from pyrit.score.float_scale.float_scale_scorer import FloatScaleScorer from pyrit.score.scorer_prompt_validator import ScorerPromptValidator @@ -21,7 +21,7 @@ class InsecureCodeScorer(FloatScaleScorer): """ _DEFAULT_VALIDATOR: ScorerPromptValidator = ScorerPromptValidator(supported_data_types=["text"]) - TARGET_REQUIREMENTS = CHAT_CONSUMER_REQUIREMENTS + TARGET_REQUIREMENTS = CHAT_TARGET_REQUIREMENTS def __init__( self, diff --git a/pyrit/score/float_scale/self_ask_general_float_scale_scorer.py b/pyrit/score/float_scale/self_ask_general_float_scale_scorer.py index 86a62eec9e..ab6c79f914 100644 --- a/pyrit/score/float_scale/self_ask_general_float_scale_scorer.py +++ b/pyrit/score/float_scale/self_ask_general_float_scale_scorer.py @@ -5,7 +5,7 @@ from typing import TYPE_CHECKING, Optional -from pyrit.prompt_target import CHAT_CONSUMER_REQUIREMENTS +from pyrit.prompt_target import CHAT_TARGET_REQUIREMENTS from pyrit.score.float_scale.float_scale_scorer import FloatScaleScorer from pyrit.score.scorer_prompt_validator import ScorerPromptValidator @@ -25,7 +25,7 @@ class SelfAskGeneralFloatScaleScorer(FloatScaleScorer): supported_data_types=["text"], is_objective_required=True, ) - TARGET_REQUIREMENTS = CHAT_CONSUMER_REQUIREMENTS + TARGET_REQUIREMENTS = CHAT_TARGET_REQUIREMENTS def __init__( self, @@ -55,7 +55,7 @@ def __init__( Args: chat_target (PromptTarget): The chat target used to score. Must satisfy - CHAT_CONSUMER_REQUIREMENTS (multi-turn + editable history capabilities, + CHAT_TARGET_REQUIREMENTS (multi-turn + editable history capabilities, possibly via normalization-pipeline adaptation). system_prompt_format_string (str): System prompt template with placeholders for objective, prompt, and message_piece. diff --git a/pyrit/score/float_scale/self_ask_likert_scorer.py b/pyrit/score/float_scale/self_ask_likert_scorer.py index 7f01ee94a6..b3ebe5543b 100644 --- a/pyrit/score/float_scale/self_ask_likert_scorer.py +++ b/pyrit/score/float_scale/self_ask_likert_scorer.py @@ -12,7 +12,7 @@ from pyrit.common.path import HARM_DEFINITION_PATH, SCORER_LIKERT_PATH from pyrit.identifiers import ComponentIdentifier from pyrit.models import MessagePiece, Score, SeedPrompt, UnvalidatedScore -from pyrit.prompt_target import CHAT_CONSUMER_REQUIREMENTS, PromptTarget +from pyrit.prompt_target import CHAT_TARGET_REQUIREMENTS, PromptTarget from pyrit.score.float_scale.float_scale_scorer import FloatScaleScorer from pyrit.score.scorer_prompt_validator import ScorerPromptValidator @@ -173,7 +173,7 @@ class SelfAskLikertScorer(FloatScaleScorer): """ _DEFAULT_VALIDATOR: ScorerPromptValidator = ScorerPromptValidator(supported_data_types=["text"]) - TARGET_REQUIREMENTS = CHAT_CONSUMER_REQUIREMENTS + TARGET_REQUIREMENTS = CHAT_TARGET_REQUIREMENTS def __init__( self, diff --git a/pyrit/score/float_scale/self_ask_scale_scorer.py b/pyrit/score/float_scale/self_ask_scale_scorer.py index 9c87d5303e..b8a5491bc0 100644 --- a/pyrit/score/float_scale/self_ask_scale_scorer.py +++ b/pyrit/score/float_scale/self_ask_scale_scorer.py @@ -11,7 +11,7 @@ from pyrit.common.path import SCORER_SCALES_PATH from pyrit.identifiers import ComponentIdentifier from pyrit.models import MessagePiece, Score, SeedPrompt, UnvalidatedScore -from pyrit.prompt_target import CHAT_CONSUMER_REQUIREMENTS, PromptTarget +from pyrit.prompt_target import CHAT_TARGET_REQUIREMENTS, PromptTarget from pyrit.score.float_scale.float_scale_scorer import FloatScaleScorer from pyrit.score.scorer_prompt_validator import ScorerPromptValidator @@ -39,7 +39,7 @@ class SystemPaths(enum.Enum): supported_data_types=["text"], is_objective_required=True, ) - TARGET_REQUIREMENTS = CHAT_CONSUMER_REQUIREMENTS + TARGET_REQUIREMENTS = CHAT_TARGET_REQUIREMENTS def __init__( self, diff --git a/pyrit/score/true_false/gandalf_scorer.py b/pyrit/score/true_false/gandalf_scorer.py index 220ac54518..4928ec7a6b 100644 --- a/pyrit/score/true_false/gandalf_scorer.py +++ b/pyrit/score/true_false/gandalf_scorer.py @@ -11,7 +11,7 @@ from pyrit.exceptions import PyritException, pyrit_target_retry from pyrit.identifiers import ComponentIdentifier from pyrit.models import Message, MessagePiece, Score -from pyrit.prompt_target import CHAT_CONSUMER_REQUIREMENTS, GandalfLevel, PromptTarget +from pyrit.prompt_target import CHAT_TARGET_REQUIREMENTS, GandalfLevel, PromptTarget from pyrit.score.scorer_prompt_validator import ScorerPromptValidator from pyrit.score.true_false.true_false_score_aggregator import ( TrueFalseAggregatorFunc, @@ -30,7 +30,7 @@ class GandalfScorer(TrueFalseScorer): """ _DEFAULT_VALIDATOR: ScorerPromptValidator = ScorerPromptValidator(supported_data_types=["text"]) - TARGET_REQUIREMENTS = CHAT_CONSUMER_REQUIREMENTS + TARGET_REQUIREMENTS = CHAT_TARGET_REQUIREMENTS def __init__( self, @@ -45,9 +45,7 @@ def __init__( Args: level (GandalfLevel): The Gandalf challenge level to score against. - chat_target (PromptTarget): The chat target to use for the scorer. Must satisfy - CHAT_CONSUMER_REQUIREMENTS (multi-turn + editable history capabilities, - possibly via normalization-pipeline adaptation). + chat_target (PromptTarget): The chat target used for password extraction. validator (Optional[ScorerPromptValidator]): Custom validator. Defaults to text data type validator. score_aggregator (TrueFalseAggregatorFunc): Aggregator for combining scores. Defaults to TrueFalseScoreAggregator.OR. diff --git a/pyrit/score/true_false/self_ask_category_scorer.py b/pyrit/score/true_false/self_ask_category_scorer.py index a9b94f479e..7505f01ecc 100644 --- a/pyrit/score/true_false/self_ask_category_scorer.py +++ b/pyrit/score/true_false/self_ask_category_scorer.py @@ -11,7 +11,7 @@ from pyrit.common.path import SCORER_CONTENT_CLASSIFIERS_PATH from pyrit.identifiers import ComponentIdentifier from pyrit.models import MessagePiece, Score, SeedPrompt, UnvalidatedScore -from pyrit.prompt_target import CHAT_CONSUMER_REQUIREMENTS, PromptTarget +from pyrit.prompt_target import CHAT_TARGET_REQUIREMENTS, PromptTarget from pyrit.score.scorer_prompt_validator import ScorerPromptValidator from pyrit.score.true_false.true_false_score_aggregator import ( TrueFalseAggregatorFunc, @@ -37,7 +37,7 @@ class SelfAskCategoryScorer(TrueFalseScorer): """ _DEFAULT_VALIDATOR: ScorerPromptValidator = ScorerPromptValidator() - TARGET_REQUIREMENTS = CHAT_CONSUMER_REQUIREMENTS + TARGET_REQUIREMENTS = CHAT_TARGET_REQUIREMENTS def __init__( self, @@ -51,9 +51,7 @@ def __init__( Initialize a new instance of the SelfAskCategoryScorer class. Args: - chat_target (PromptTarget): The chat target to use for the scorer. Must satisfy - CHAT_CONSUMER_REQUIREMENTS (multi-turn + editable history capabilities, - possibly via normalization-pipeline adaptation). + chat_target (PromptTarget): The chat target to interact with. content_classifier_path (Union[str, Path]): The path to the classifier YAML file. score_aggregator (TrueFalseAggregatorFunc): The aggregator function to use. Defaults to TrueFalseScoreAggregator.OR. diff --git a/pyrit/score/true_false/self_ask_general_true_false_scorer.py b/pyrit/score/true_false/self_ask_general_true_false_scorer.py index 893519b064..be9465554e 100644 --- a/pyrit/score/true_false/self_ask_general_true_false_scorer.py +++ b/pyrit/score/true_false/self_ask_general_true_false_scorer.py @@ -5,7 +5,7 @@ from typing import TYPE_CHECKING, Optional -from pyrit.prompt_target import CHAT_CONSUMER_REQUIREMENTS +from pyrit.prompt_target import CHAT_TARGET_REQUIREMENTS from pyrit.score.scorer_prompt_validator import ScorerPromptValidator from pyrit.score.true_false.true_false_score_aggregator import ( TrueFalseAggregatorFunc, @@ -29,7 +29,7 @@ class SelfAskGeneralTrueFalseScorer(TrueFalseScorer): supported_data_types=["text"], is_objective_required=False, ) - TARGET_REQUIREMENTS = CHAT_CONSUMER_REQUIREMENTS + TARGET_REQUIREMENTS = CHAT_TARGET_REQUIREMENTS def __init__( self, @@ -58,7 +58,7 @@ def __init__( Args: chat_target (PromptTarget): The chat target used to score. Must satisfy - CHAT_CONSUMER_REQUIREMENTS (multi-turn + editable history capabilities, + CHAT_TARGET_REQUIREMENTS (multi-turn + editable history capabilities, possibly via normalization-pipeline adaptation). system_prompt_format_string (str): System prompt template with placeholders for objective, task (alias of objective), prompt, and message_piece. diff --git a/pyrit/score/true_false/self_ask_question_answer_scorer.py b/pyrit/score/true_false/self_ask_question_answer_scorer.py index c6d35cdfe8..d5a4471075 100644 --- a/pyrit/score/true_false/self_ask_question_answer_scorer.py +++ b/pyrit/score/true_false/self_ask_question_answer_scorer.py @@ -47,7 +47,7 @@ def __init__( Args: chat_target (PromptTarget): The chat target to use for the scorer. Must satisfy - CHAT_CONSUMER_REQUIREMENTS (multi-turn + editable history capabilities, + CHAT_TARGET_REQUIREMENTS (multi-turn + editable history capabilities, possibly via normalization-pipeline adaptation). true_false_question_path (Optional[pathlib.Path]): The path to the true/false question file. Defaults to None, which uses the default question_answering.yaml file. diff --git a/pyrit/score/true_false/self_ask_refusal_scorer.py b/pyrit/score/true_false/self_ask_refusal_scorer.py index fae360ad6e..26cfc8e235 100644 --- a/pyrit/score/true_false/self_ask_refusal_scorer.py +++ b/pyrit/score/true_false/self_ask_refusal_scorer.py @@ -8,7 +8,7 @@ from pyrit.common.path import SCORER_SEED_PROMPT_PATH from pyrit.identifiers import ComponentIdentifier from pyrit.models import MessagePiece, Score, SeedPrompt, UnvalidatedScore -from pyrit.prompt_target import CHAT_CONSUMER_REQUIREMENTS, PromptTarget +from pyrit.prompt_target import CHAT_TARGET_REQUIREMENTS, PromptTarget from pyrit.score.scorer_prompt_validator import ScorerPromptValidator from pyrit.score.true_false.true_false_score_aggregator import ( TrueFalseAggregatorFunc, @@ -64,7 +64,7 @@ class SelfAskRefusalScorer(TrueFalseScorer): """ _DEFAULT_VALIDATOR: ScorerPromptValidator = ScorerPromptValidator() - TARGET_REQUIREMENTS = CHAT_CONSUMER_REQUIREMENTS + TARGET_REQUIREMENTS = CHAT_TARGET_REQUIREMENTS def __init__( self, @@ -80,7 +80,7 @@ def __init__( Args: chat_target (PromptTarget): The chat target to use for the scorer. Must satisfy - CHAT_CONSUMER_REQUIREMENTS (multi-turn + editable history capabilities, + CHAT_TARGET_REQUIREMENTS (multi-turn + editable history capabilities, possibly via normalization-pipeline adaptation). refusal_system_prompt_path (Union[RefusalScorerPaths, Path, str]): The path to the system prompt to use for refusal detection. Can be a RefusalScorerPaths enum value, a Path, or a string path. diff --git a/pyrit/score/true_false/self_ask_true_false_scorer.py b/pyrit/score/true_false/self_ask_true_false_scorer.py index a27acdcf24..193b0519af 100644 --- a/pyrit/score/true_false/self_ask_true_false_scorer.py +++ b/pyrit/score/true_false/self_ask_true_false_scorer.py @@ -12,7 +12,7 @@ from pyrit.common.path import SCORER_SEED_PROMPT_PATH from pyrit.identifiers import ComponentIdentifier from pyrit.models import MessagePiece, Score, SeedPrompt -from pyrit.prompt_target import CHAT_CONSUMER_REQUIREMENTS, PromptTarget +from pyrit.prompt_target import CHAT_TARGET_REQUIREMENTS, PromptTarget from pyrit.score.scorer_prompt_validator import ScorerPromptValidator from pyrit.score.true_false.true_false_score_aggregator import ( TrueFalseAggregatorFunc, @@ -100,7 +100,7 @@ class SelfAskTrueFalseScorer(TrueFalseScorer): _DEFAULT_VALIDATOR: ScorerPromptValidator = ScorerPromptValidator( supported_data_types=["text", "image_path"], ) - TARGET_REQUIREMENTS = CHAT_CONSUMER_REQUIREMENTS + TARGET_REQUIREMENTS = CHAT_TARGET_REQUIREMENTS def __init__( self, @@ -117,7 +117,7 @@ def __init__( Args: chat_target (PromptTarget): The chat target to use for the scorer. Must satisfy - CHAT_CONSUMER_REQUIREMENTS (multi-turn + editable history capabilities, + CHAT_TARGET_REQUIREMENTS (multi-turn + editable history capabilities, possibly via normalization-pipeline adaptation). true_false_question_path (Optional[Union[str, Path]]): The path to the true/false question file. true_false_question (Optional[TrueFalseQuestion]): The true/false question object. diff --git a/pyrit/setup/initializers/components/scorers.py b/pyrit/setup/initializers/components/scorers.py index 3b2d3aee6c..c0da388d26 100644 --- a/pyrit/setup/initializers/components/scorers.py +++ b/pyrit/setup/initializers/components/scorers.py @@ -45,7 +45,7 @@ from pyrit.setup.initializers.pyrit_initializer import PyRITInitializer if TYPE_CHECKING: - from pyrit.prompt_target.common.prompt_chat_target import PromptChatTarget + from pyrit.prompt_target import PromptTarget logger = logging.getLogger(__name__) RequiredDependencyT = TypeVar("RequiredDependencyT") @@ -572,12 +572,12 @@ def _get_scorer_registry(self) -> ScorerRegistry: """ return ScorerRegistry.get_registry_singleton() - def _get_chat_target(self, target_name: str) -> "PromptChatTarget | None": + def _get_chat_target(self, target_name: str) -> "PromptTarget | None": """ Get a chat target from the singleton target registry by name. Returns: - PromptChatTarget | None: The chat target instance if found, otherwise None. + PromptTarget | None: The chat target instance if found, otherwise None. """ target_registry = TargetRegistry.get_registry_singleton() return target_registry.get_instance_by_name(target_name) diff --git a/tests/integration/mocks.py b/tests/integration/mocks.py index dd0e187bac..f4787507b1 100644 --- a/tests/integration/mocks.py +++ b/tests/integration/mocks.py @@ -9,7 +9,7 @@ from pyrit.identifiers import ComponentIdentifier from pyrit.memory import MemoryInterface, SQLiteMemory from pyrit.models import Message, MessagePiece -from pyrit.prompt_target import PromptChatTarget, limit_requests_per_minute +from pyrit.prompt_target import PromptTarget, TargetCapabilities, TargetConfiguration, limit_requests_per_minute def get_memory_interface() -> Generator[MemoryInterface, None, None]: @@ -36,7 +36,16 @@ def get_sqlite_memory() -> Generator[SQLiteMemory, None, None]: sqlite_memory.dispose_engine() -class MockPromptTarget(PromptChatTarget): +class MockPromptTarget(PromptTarget): + _DEFAULT_CONFIGURATION: TargetConfiguration = TargetConfiguration( + capabilities=TargetCapabilities( + supports_multi_turn=True, + supports_multi_message_pieces=True, + supports_system_prompt=True, + supports_editable_history=True, + ) + ) + prompt_sent: list[str] def __init__(self, id=None, rpm=None) -> None: # noqa: A002 diff --git a/tests/integration/targets/test_openai_chat_target_integration.py b/tests/integration/targets/test_openai_chat_target_integration.py index d80f32b2bc..5ae43605d9 100644 --- a/tests/integration/targets/test_openai_chat_target_integration.py +++ b/tests/integration/targets/test_openai_chat_target_integration.py @@ -17,9 +17,7 @@ from pyrit.common.path import HOME_PATH from pyrit.models import MessagePiece -from pyrit.prompt_target import OpenAIChatAudioConfig, OpenAIChatTarget -from pyrit.prompt_target.common.target_capabilities import TargetCapabilities -from pyrit.prompt_target.common.target_configuration import TargetConfiguration +from pyrit.prompt_target import OpenAIChatAudioConfig, OpenAIChatTarget, TargetCapabilities, TargetConfiguration # Path to sample audio file for testing SAMPLE_AUDIO_FILE = HOME_PATH / "assets" / "converted_audio.wav" diff --git a/tests/partner_integration/azure_ai_evaluation/test_import_smoke.py b/tests/partner_integration/azure_ai_evaluation/test_import_smoke.py index 1bcd839b93..008136ac99 100644 --- a/tests/partner_integration/azure_ai_evaluation/test_import_smoke.py +++ b/tests/partner_integration/azure_ai_evaluation/test_import_smoke.py @@ -52,29 +52,6 @@ def test_redteam_public_api_imports(self): assert SupportedLanguages is not None -class TestPromptChatTargetTransitionalCompat: - """Verify PromptChatTarget still exists and extends PromptTarget. - - The SDK currently imports PromptChatTarget in 6+ production files - (_callback_chat_target.py, _orchestrator_manager.py, _scenario_orchestrator.py, - _execution_manager.py, strategy_utils.py, _rai_service_target.py). PyRIT is - migrating from PromptChatTarget to PromptTarget, but during the transition - both must exist with correct inheritance. - """ - - def test_prompt_chat_target_exists(self): - """PromptChatTarget must remain importable during the transition.""" - from pyrit.prompt_target import PromptChatTarget - - assert PromptChatTarget is not None - - def test_prompt_chat_target_extends_prompt_target(self): - """PromptChatTarget must be a subclass of PromptTarget.""" - from pyrit.prompt_target import PromptChatTarget - - assert issubclass(PromptChatTarget, PromptTarget) - - @requires_azure_ai_evaluation class TestCallbackChatTargetInheritance: """Verify _CallbackChatTarget correctly extends PromptTarget. diff --git a/tests/unit/backend/test_converter_service.py b/tests/unit/backend/test_converter_service.py index 3e2f8ce366..90fc070191 100644 --- a/tests/unit/backend/test_converter_service.py +++ b/tests/unit/backend/test_converter_service.py @@ -385,7 +385,7 @@ def _try_instantiate_converter(converter_name: str): """ Try to instantiate a converter with minimal representative arguments. - Uses mock objects for complex dependencies (PromptChatTarget, PromptConverter) + Uses mock objects for complex dependencies (PromptTarget, PromptConverter) and provides minimal valid values for simple required parameters so that the identifier extraction test covers ALL converters without skipping. @@ -400,8 +400,7 @@ def _try_instantiate_converter(converter_name: str): from unittest.mock import MagicMock from pyrit.common.apply_defaults import _RequiredValueSentinel - from pyrit.prompt_target.common.prompt_chat_target import PromptChatTarget - from pyrit.prompt_target.common.prompt_target import PromptTarget + from pyrit.prompt_target import PromptTarget # Converters requiring external credentials or resources that can't be mocked # at the constructor level — these validate env vars / files in __init__ body @@ -450,18 +449,11 @@ def _try_instantiate_converter(converter_name: str): ann = param.annotation ann_str = str(ann) if ann is not inspect.Parameter.empty else "" - # PromptChatTarget / PromptTarget — mock it with a proper identifier + # PromptTarget — mock it with a proper identifier if ann is not inspect.Parameter.empty and ( - (isinstance(ann, type) and issubclass(ann, PromptTarget)) - or "PromptChatTarget" in ann_str - or "PromptTarget" in ann_str + (isinstance(ann, type) and issubclass(ann, PromptTarget)) or "PromptTarget" in ann_str ): - spec_cls = ( - PromptChatTarget - if (isinstance(ann, type) and issubclass(ann, PromptChatTarget)) or "PromptChatTarget" in ann_str - else PromptTarget - ) - mock_target = MagicMock(spec=spec_cls) + mock_target = MagicMock(spec=PromptTarget) mock_target.__class__.__name__ = "MockChatTarget" # Configure get_identifier() to return a proper identifier-like object # so that _create_identifier can extract class_name, model_name, etc. @@ -534,7 +526,7 @@ def test_build_instance_from_converter(self, converter_name: str) -> None: Test that _build_instance_from_object works with each converter. Instantiates every converter with minimal representative arguments - (using mocks for complex dependencies like PromptChatTarget) and verifies: + (using mocks for complex dependencies like PromptTarget) and verifies: - converter_id is set correctly - converter_type matches the class name - supported_input_types and supported_output_types are lists diff --git a/tests/unit/executor/attack/component/test_conversation_manager.py b/tests/unit/executor/attack/component/test_conversation_manager.py index 292252fe44..c0bae706b5 100644 --- a/tests/unit/executor/attack/component/test_conversation_manager.py +++ b/tests/unit/executor/attack/component/test_conversation_manager.py @@ -37,7 +37,7 @@ from pyrit.identifiers import ComponentIdentifier from pyrit.models import Message, MessagePiece, Score from pyrit.prompt_normalizer import PromptConverterConfiguration, PromptNormalizer -from pyrit.prompt_target import PromptChatTarget, PromptTarget +from pyrit.prompt_target import PromptTarget def _mock_target_id(name: str = "MockTarget") -> ComponentIdentifier: @@ -85,9 +85,11 @@ def mock_prompt_normalizer() -> MagicMock: @pytest.fixture def mock_chat_target() -> MagicMock: """Create a mock chat target for testing.""" - target = MagicMock(spec=PromptChatTarget) + target = MagicMock(spec=PromptTarget) target.set_system_prompt = MagicMock() target.get_identifier.return_value = _mock_target_id("MockChatTarget") + target.capabilities.supports_multi_turn = True + target.capabilities.supports_editable_history = True return target @@ -102,6 +104,8 @@ def mock_prompt_target() -> MagicMock: capabilities=TargetCapabilities(supports_multi_turn=False, supports_system_prompt=False), ) target.get_identifier.return_value = _mock_target_id("MockTarget") + target.capabilities.supports_multi_turn = False + target.capabilities.supports_editable_history = False return target @@ -1087,7 +1091,9 @@ async def test_non_chat_target_behavior_raise_explicit( config = PrependedConversationConfig(non_chat_target_behavior="raise") with pytest.raises( - ValueError, match="prepended_conversation requires the objective target to be a chat-capable" + ValueError, + match="prepended_conversation requires the objective target to support multi-turn conversations" + " with editable history", ): await manager.initialize_context_async( context=context, diff --git a/tests/unit/executor/attack/component/test_simulated_conversation.py b/tests/unit/executor/attack/component/test_simulated_conversation.py index 7e5d1d9334..ee5d85cc53 100644 --- a/tests/unit/executor/attack/component/test_simulated_conversation.py +++ b/tests/unit/executor/attack/component/test_simulated_conversation.py @@ -22,7 +22,7 @@ SeedPrompt, SimulatedTargetSystemPromptPaths, ) -from pyrit.prompt_target import PromptChatTarget +from pyrit.prompt_target import PromptTarget from pyrit.score import TrueFalseScorer @@ -45,7 +45,7 @@ def _mock_scorer_id(name: str = "MockScorer") -> ComponentIdentifier: @pytest.fixture def mock_adversarial_chat() -> MagicMock: """Create a mock adversarial chat target for testing.""" - chat = MagicMock(spec=PromptChatTarget) + chat = MagicMock(spec=PromptTarget) chat.send_prompt_async = AsyncMock() chat.set_system_prompt = MagicMock() chat.get_identifier.return_value = _mock_target_id("MockAdversarialChat") diff --git a/tests/unit/executor/attack/multi_turn/test_crescendo.py b/tests/unit/executor/attack/multi_turn/test_crescendo.py index bf79014f31..5f3b157ef0 100644 --- a/tests/unit/executor/attack/multi_turn/test_crescendo.py +++ b/tests/unit/executor/attack/multi_turn/test_crescendo.py @@ -35,7 +35,7 @@ ScoreType, ) from pyrit.prompt_normalizer import PromptNormalizer -from pyrit.prompt_target import PromptChatTarget +from pyrit.prompt_target import PromptTarget from pyrit.score import FloatScaleThresholdScorer, SelfAskRefusalScorer, TrueFalseScorer from pyrit.score.score_utils import ORIGINAL_FLOAT_VALUE_KEY @@ -62,7 +62,7 @@ def create_mock_chat_target(*, name: str = "MockChatTarget") -> MagicMock: This standardizes the creation of mock chat targets across tests, ensuring they all have the required methods and return values. """ - target = MagicMock(spec=PromptChatTarget) + target = MagicMock(spec=PromptTarget) target.send_prompt_async = AsyncMock() target.set_system_prompt = MagicMock() target.get_identifier.return_value = _mock_target_id(name) diff --git a/tests/unit/executor/attack/multi_turn/test_red_teaming.py b/tests/unit/executor/attack/multi_turn/test_red_teaming.py index 7566f8efe3..b31c39c1e0 100644 --- a/tests/unit/executor/attack/multi_turn/test_red_teaming.py +++ b/tests/unit/executor/attack/multi_turn/test_red_teaming.py @@ -31,7 +31,7 @@ SeedPrompt, ) from pyrit.prompt_normalizer import PromptNormalizer -from pyrit.prompt_target import PromptChatTarget, PromptTarget +from pyrit.prompt_target import PromptTarget from pyrit.score import Scorer, TrueFalseScorer @@ -61,7 +61,7 @@ def mock_objective_target() -> MagicMock: @pytest.fixture def mock_adversarial_chat() -> MagicMock: - chat = MagicMock(spec=PromptChatTarget) + chat = MagicMock(spec=PromptTarget) chat.send_prompt_async = AsyncMock() chat.set_system_prompt = MagicMock() chat.get_identifier.return_value = _mock_target_id("MockChatTarget") @@ -556,8 +556,8 @@ async def test_max_turns_validation_with_prepended_conversation( mock_adversarial_chat: MagicMock, ): """Test that prepended conversation turns are validated against max_turns.""" - # Create a separate chat target for objective since prepended_conversation requires PromptChatTarget - mock_chat_objective_target = MagicMock(spec=PromptChatTarget) + # Create a separate chat target for objective since prepended_conversation requires PromptTarget + mock_chat_objective_target = MagicMock(spec=PromptTarget) mock_chat_objective_target.send_prompt_async = AsyncMock() mock_chat_objective_target.set_system_prompt = MagicMock() mock_chat_objective_target.get_identifier.return_value = _mock_target_id("MockChatTarget") diff --git a/tests/unit/executor/attack/multi_turn/test_tree_of_attacks.py b/tests/unit/executor/attack/multi_turn/test_tree_of_attacks.py index 6c7c6d7836..94070abd75 100644 --- a/tests/unit/executor/attack/multi_turn/test_tree_of_attacks.py +++ b/tests/unit/executor/attack/multi_turn/test_tree_of_attacks.py @@ -37,8 +37,7 @@ SeedPrompt, ) from pyrit.prompt_normalizer import PromptNormalizer -from pyrit.prompt_target import PromptChatTarget, PromptTarget -from pyrit.prompt_target.common.target_capabilities import CapabilityName +from pyrit.prompt_target import CapabilityName, PromptTarget from pyrit.score import FloatScaleThresholdScorer, Scorer, TrueFalseScorer from pyrit.score.float_scale.float_scale_scorer import FloatScaleScorer from pyrit.score.score_utils import normalize_score_to_float @@ -152,7 +151,7 @@ class AttackBuilder: def __init__(self) -> None: self.objective_target: Optional[PromptTarget] = None - self.adversarial_chat: Optional[PromptChatTarget] = None + self.adversarial_chat: Optional[PromptTarget] = None self.objective_scorer: Optional[Scorer] = None self.auxiliary_scorers: list[Scorer] = [] self.tree_params: dict[str, Any] = {} @@ -253,15 +252,15 @@ def _create_mock_target(supports_multi_turn: bool = True) -> PromptTarget: return cast("PromptTarget", target) @staticmethod - def _create_mock_chat() -> PromptChatTarget: - chat = MagicMock(spec=PromptChatTarget) + def _create_mock_chat() -> PromptTarget: + chat = MagicMock(spec=PromptTarget) chat.send_prompt_async = AsyncMock(return_value=None) chat.set_system_prompt = MagicMock() chat.get_identifier.return_value = ComponentIdentifier( class_name="MockChatTarget", class_module="test_module", ) - return cast("PromptChatTarget", chat) + return cast("PromptTarget", chat) @staticmethod def _create_mock_scorer(name: str) -> TrueFalseScorer: @@ -2248,9 +2247,9 @@ def test_tap_init_raises_when_objective_scorer_is_none(): scoring_config = AttackScoringConfig(objective_scorer=None) with pytest.raises(ValueError, match="objective_scorer is required"): TreeOfAttacksWithPruningAttack( - objective_target=MagicMock(spec=PromptChatTarget), + objective_target=MagicMock(spec=PromptTarget), attack_adversarial_config=MagicMock( - target=MagicMock(spec=PromptChatTarget), + target=MagicMock(spec=PromptTarget), system_prompt_path=None, ), attack_scoring_config=scoring_config, diff --git a/tests/unit/executor/attack/single_turn/test_context_compliance.py b/tests/unit/executor/attack/single_turn/test_context_compliance.py index ba2314e90f..d9f506c693 100644 --- a/tests/unit/executor/attack/single_turn/test_context_compliance.py +++ b/tests/unit/executor/attack/single_turn/test_context_compliance.py @@ -23,7 +23,7 @@ SeedPrompt, ) from pyrit.prompt_normalizer import PromptNormalizer -from pyrit.prompt_target import PromptChatTarget +from pyrit.prompt_target import PromptTarget from pyrit.score import TrueFalseScorer @@ -45,8 +45,8 @@ def _mock_scorer_id(name: str = "MockScorer") -> ComponentIdentifier: @pytest.fixture def mock_objective_target(): - """Create a mock PromptChatTarget for testing""" - target = MagicMock(spec=PromptChatTarget) + """Create a mock PromptTarget for testing""" + target = MagicMock(spec=PromptTarget) target.send_prompt_async = AsyncMock() target.get_identifier.return_value = _mock_target_id("MockTarget") return target @@ -55,7 +55,7 @@ def mock_objective_target(): @pytest.fixture def mock_adversarial_chat(): """Create a mock adversarial chat target for testing""" - target = MagicMock(spec=PromptChatTarget) + target = MagicMock(spec=PromptTarget) target.send_prompt_async = AsyncMock() target.get_identifier.return_value = _mock_target_id("MockAdversarialTarget") return target diff --git a/tests/unit/executor/attack/single_turn/test_flip_attack.py b/tests/unit/executor/attack/single_turn/test_flip_attack.py index cf35267116..d488eec5e8 100644 --- a/tests/unit/executor/attack/single_turn/test_flip_attack.py +++ b/tests/unit/executor/attack/single_turn/test_flip_attack.py @@ -20,7 +20,7 @@ ) from pyrit.prompt_converter import FlipConverter from pyrit.prompt_normalizer import PromptConverterConfiguration, PromptNormalizer -from pyrit.prompt_target import PromptChatTarget +from pyrit.prompt_target import PromptTarget from pyrit.score import TrueFalseScorer @@ -42,8 +42,8 @@ def _mock_scorer_id(name: str = "MockScorer") -> ComponentIdentifier: @pytest.fixture def mock_objective_target(): - """Create a mock PromptChatTarget for testing""" - target = MagicMock(spec=PromptChatTarget) + """Create a mock PromptTarget for testing""" + target = MagicMock(spec=PromptTarget) target.send_prompt_async = AsyncMock() target.get_identifier.return_value = _mock_target_id("MockTarget") return target diff --git a/tests/unit/executor/attack/single_turn/test_role_play.py b/tests/unit/executor/attack/single_turn/test_role_play.py index eba6b77fd8..eda4782e1f 100644 --- a/tests/unit/executor/attack/single_turn/test_role_play.py +++ b/tests/unit/executor/attack/single_turn/test_role_play.py @@ -25,14 +25,14 @@ ) from pyrit.prompt_converter import Base64Converter, StringJoinConverter from pyrit.prompt_normalizer import PromptConverterConfiguration -from pyrit.prompt_target import PromptChatTarget +from pyrit.prompt_target import PromptTarget from pyrit.score import Scorer, TrueFalseScorer @pytest.fixture def mock_objective_target(): """Create a mock prompt target for testing""" - target = MagicMock(spec=PromptChatTarget) + target = MagicMock(spec=PromptTarget) target.send_prompt_async = AsyncMock() target.get_identifier.return_value = get_mock_target_identifier("MockTarget") return target @@ -41,7 +41,7 @@ def mock_objective_target(): @pytest.fixture def mock_adversarial_chat_target(): """Create a mock adversarial chat target for testing""" - target = MagicMock(spec=PromptChatTarget) + target = MagicMock(spec=PromptTarget) target.send_prompt_async = AsyncMock() target.get_identifier.return_value = get_mock_target_identifier("MockAdversarialChat") return target diff --git a/tests/unit/executor/attack/test_attack_parameter_consistency.py b/tests/unit/executor/attack/test_attack_parameter_consistency.py index 4241ca1342..6c56018892 100644 --- a/tests/unit/executor/attack/test_attack_parameter_consistency.py +++ b/tests/unit/executor/attack/test_attack_parameter_consistency.py @@ -34,7 +34,7 @@ Score, ) from pyrit.prompt_normalizer import PromptNormalizer -from pyrit.prompt_target import PromptChatTarget, PromptTarget +from pyrit.prompt_target import PromptTarget from pyrit.score import FloatScaleThresholdScorer, TrueFalseScorer @@ -140,8 +140,8 @@ def prepended_conversation_multimodal() -> list[Message]: @pytest.fixture def mock_chat_target() -> MagicMock: - """Create a mock PromptChatTarget with common setup.""" - target = MagicMock(spec=PromptChatTarget) + """Create a mock PromptTarget with common setup.""" + target = MagicMock(spec=PromptTarget) target.send_prompt_async = AsyncMock() target.set_system_prompt = MagicMock() target.get_identifier.return_value = _mock_target_id("MockChatTarget") @@ -160,7 +160,7 @@ def mock_non_chat_target() -> MagicMock: @pytest.fixture def mock_adversarial_chat() -> MagicMock: """Create a mock adversarial chat target.""" - target = MagicMock(spec=PromptChatTarget) + target = MagicMock(spec=PromptTarget) target.send_prompt_async = AsyncMock() target.set_system_prompt = MagicMock() target.get_identifier.return_value = _mock_target_id("MockAdversarialChat") @@ -560,7 +560,7 @@ class TestPrependedConversationInMemory: """ Tests verifying that prepended_conversation is properly added to memory. - For PromptChatTargets, prepended_conversation should: + For chat style PromptTargets, prepended_conversation should: 1. Be added to memory with the correct conversation_id 2. Have assistant messages translated to simulated_assistant role 3. Preserve multi-modal content diff --git a/tests/unit/executor/attack/test_error_skip_scoring.py b/tests/unit/executor/attack/test_error_skip_scoring.py index 2ab5575a42..8d076151e0 100644 --- a/tests/unit/executor/attack/test_error_skip_scoring.py +++ b/tests/unit/executor/attack/test_error_skip_scoring.py @@ -24,7 +24,6 @@ from pyrit.identifiers import ComponentIdentifier from pyrit.models import Message, MessagePiece, SeedGroup, SeedPrompt from pyrit.prompt_target import PromptTarget -from pyrit.prompt_target.common.prompt_chat_target import PromptChatTarget from pyrit.score import FloatScaleThresholdScorer, TrueFalseScorer @@ -181,9 +180,9 @@ async def test_attack_executor_skips_scoring_on_error( # Setup additional configs for multi-turn attacks that need adversarial config if attack_class in [RedTeamingAttack, CrescendoAttack, TreeOfAttacksWithPruningAttack]: - # TreeOfAttacks requires PromptChatTarget, others can use PromptTarget + # TreeOfAttacks requires PromptTarget, others can use PromptTarget if attack_class == TreeOfAttacksWithPruningAttack: - adversarial_target = MagicMock(spec=PromptChatTarget) + adversarial_target = MagicMock(spec=PromptTarget) else: adversarial_target = MagicMock(spec=PromptTarget) diff --git a/tests/unit/executor/promptgen/test_anecdoctor.py b/tests/unit/executor/promptgen/test_anecdoctor.py index 377f449dae..7c63d70859 100644 --- a/tests/unit/executor/promptgen/test_anecdoctor.py +++ b/tests/unit/executor/promptgen/test_anecdoctor.py @@ -15,7 +15,7 @@ from pyrit.identifiers import ComponentIdentifier from pyrit.models import Message from pyrit.prompt_normalizer import PromptNormalizer -from pyrit.prompt_target import PromptChatTarget +from pyrit.prompt_target import PromptTarget def _mock_target_id(name: str = "MockTarget") -> ComponentIdentifier: @@ -27,18 +27,18 @@ def _mock_target_id(name: str = "MockTarget") -> ComponentIdentifier: @pytest.fixture -def mock_objective_target() -> PromptChatTarget: +def mock_objective_target() -> PromptTarget: """Create a mock objective target for testing.""" - mock_target = MagicMock(spec=PromptChatTarget) + mock_target = MagicMock(spec=PromptTarget) mock_target.set_system_prompt = MagicMock() mock_target.get_identifier.return_value = _mock_target_id("mock_objective_target") return mock_target @pytest.fixture -def mock_processing_model() -> PromptChatTarget: +def mock_processing_model() -> PromptTarget: """Create a mock processing model for testing.""" - mock_model = MagicMock(spec=PromptChatTarget) + mock_model = MagicMock(spec=PromptTarget) mock_model.set_system_prompt = MagicMock() mock_model.get_identifier.return_value = _mock_target_id("MockProcessingModel") return mock_model @@ -527,7 +527,7 @@ def test_special_characters_in_data(self, mock_objective_target): @pytest.mark.usefixtures("patch_central_database") async def test_extract_knowledge_graph_raises_when_processing_model_is_none(): """Test that _extract_knowledge_graph_async raises ValueError when processing model is None.""" - mock_target = MagicMock(spec=PromptChatTarget) + mock_target = MagicMock(spec=PromptTarget) mock_target.get_identifier.return_value = ComponentIdentifier(class_name="MockTarget", class_module="test_module") generator = AnecdoctorGenerator(objective_target=mock_target) # Ensure processing model is explicitly None diff --git a/tests/unit/mocks.py b/tests/unit/mocks.py index 84850dfc5d..50c07547f4 100644 --- a/tests/unit/mocks.py +++ b/tests/unit/mocks.py @@ -13,7 +13,7 @@ from pyrit.identifiers import ComponentIdentifier from pyrit.memory import AzureSQLMemory, CentralMemory, PromptMemoryEntry from pyrit.models import Message, MessagePiece -from pyrit.prompt_target import PromptChatTarget, PromptTarget, limit_requests_per_minute +from pyrit.prompt_target import PromptTarget, TargetCapabilities, TargetConfiguration, limit_requests_per_minute def get_mock_scorer_identifier() -> ComponentIdentifier: @@ -119,7 +119,16 @@ def raise_for_status(self): raise Exception(f"HTTP Error {self.status}") -class MockPromptTarget(PromptChatTarget): +class MockPromptTarget(PromptTarget): + _DEFAULT_CONFIGURATION: TargetConfiguration = TargetConfiguration( + capabilities=TargetCapabilities( + supports_multi_turn=True, + supports_multi_message_pieces=True, + supports_system_prompt=True, + supports_editable_history=True, + ) + ) + prompt_sent: list[str] def __init__(self, id=None, rpm=None) -> None: # noqa: A002 diff --git a/tests/unit/prompt_converter/test_toxic_sentence_generator_converter.py b/tests/unit/prompt_converter/test_toxic_sentence_generator_converter.py index c7309785e1..22b7fb46eb 100644 --- a/tests/unit/prompt_converter/test_toxic_sentence_generator_converter.py +++ b/tests/unit/prompt_converter/test_toxic_sentence_generator_converter.py @@ -8,12 +8,12 @@ from pyrit.models import MessagePiece, SeedPrompt from pyrit.prompt_converter import ToxicSentenceGeneratorConverter -from pyrit.prompt_target import PromptChatTarget +from pyrit.prompt_target import PromptTarget @pytest.fixture def mock_target(): - mock = MagicMock(spec=PromptChatTarget) + mock = MagicMock(spec=PromptTarget) # Create a Message response instead of PromptResponse response = MessagePiece( role="assistant", @@ -72,7 +72,7 @@ async def test_toxic_sentence_generator_convert(mock_target, mock_template): async def test_toxic_sentence_generator_input_output_supported(): """Test that the converter correctly identifies supported input/output types.""" with patch("pyrit.prompt_converter.toxic_sentence_generator_converter.SeedPrompt.from_yaml_file"): - converter = ToxicSentenceGeneratorConverter(converter_target=MagicMock(spec=PromptChatTarget)) + converter = ToxicSentenceGeneratorConverter(converter_target=MagicMock(spec=PromptTarget)) assert converter.input_supported("text") is True assert converter.input_supported("image") is False diff --git a/tests/unit/prompt_target/target/test_openai_chat_target.py b/tests/unit/prompt_target/target/test_openai_chat_target.py index 59395a270f..174bb54ed9 100644 --- a/tests/unit/prompt_target/target/test_openai_chat_target.py +++ b/tests/unit/prompt_target/target/test_openai_chat_target.py @@ -32,7 +32,7 @@ OpenAIChatAudioConfig, OpenAIChatTarget, OpenAIResponseTarget, - PromptChatTarget, + PromptTarget, ) from pyrit.prompt_target.common.target_capabilities import TargetCapabilities from pyrit.prompt_target.common.target_configuration import TargetConfiguration @@ -575,19 +575,21 @@ def test_validate_request_unsupported_data_types(target: OpenAIChatTarget): os.remove(image_piece.original_value) -def test_inheritance_from_prompt_chat_target(target: OpenAIChatTarget): - """Test that OpenAIChatTarget properly inherits from PromptChatTarget.""" - assert isinstance(target, PromptChatTarget), "OpenAIChatTarget must inherit from PromptChatTarget" +def test_inheritance_from_prompt_target(target: OpenAIChatTarget): + """OpenAIChatTarget inherits from PromptTarget and declares chat capabilities.""" + assert isinstance(target, PromptTarget), "OpenAIChatTarget must inherit from PromptTarget" + assert target.capabilities.supports_multi_turn is True + assert target.capabilities.supports_editable_history is True -def test_inheritance_from_prompt_chat_target_base(): - """Test that OpenAIChatTargetBase properly inherits from PromptChatTarget.""" - - # Create a minimal instance to test inheritance +def test_inheritance_from_prompt_target_base(): + """OpenAIChatTarget (via OpenAIChatTargetBase) inherits from PromptTarget.""" target = OpenAIChatTarget(model_name="test-model", endpoint="https://test.com", api_key="test-key") - assert isinstance(target, PromptChatTarget), ( - "OpenAIChatTarget must inherit from PromptChatTarget through OpenAIChatTargetBase" + assert isinstance(target, PromptTarget), ( + "OpenAIChatTarget must inherit from PromptTarget through OpenAIChatTargetBase" ) + assert target.capabilities.supports_multi_turn is True + assert target.capabilities.supports_editable_history is True def test_is_response_format_json_supported(target: OpenAIChatTarget): diff --git a/tests/unit/prompt_target/target/test_openai_response_target.py b/tests/unit/prompt_target/target/test_openai_response_target.py index 95f6e238f8..fca98f41a2 100644 --- a/tests/unit/prompt_target/target/test_openai_response_target.py +++ b/tests/unit/prompt_target/target/test_openai_response_target.py @@ -26,7 +26,7 @@ from pyrit.memory.memory_interface import MemoryInterface from pyrit.models import Message, MessagePiece from pyrit.models.json_response_config import _JsonResponseConfig -from pyrit.prompt_target import OpenAIResponseTarget, PromptChatTarget +from pyrit.prompt_target import OpenAIResponseTarget, PromptTarget def create_mock_response(response_dict: dict = None) -> MagicMock: @@ -579,9 +579,11 @@ def test_validate_request_unsupported_data_types(target: OpenAIResponseTarget): os.remove(image_piece.original_value) -def test_inheritance_from_prompt_chat_target(target: OpenAIResponseTarget): - """Test that OpenAIResponseTarget properly inherits from PromptChatTarget.""" - assert isinstance(target, PromptChatTarget), "OpenAIResponseTarget must inherit from PromptChatTarget" +def test_inheritance_from_prompt_target(target: OpenAIResponseTarget): + """OpenAIResponseTarget inherits from PromptTarget and declares chat capabilities.""" + assert isinstance(target, PromptTarget), "OpenAIResponseTarget must inherit from PromptTarget" + assert target.capabilities.supports_multi_turn is True + assert target.capabilities.supports_editable_history is True def test_is_response_format_json_supported(target: OpenAIResponseTarget): diff --git a/tests/unit/prompt_target/target/test_target_capabilities.py b/tests/unit/prompt_target/target/test_target_capabilities.py index 502efadadd..c8e6de4c3a 100644 --- a/tests/unit/prompt_target/target/test_target_capabilities.py +++ b/tests/unit/prompt_target/target/test_target_capabilities.py @@ -537,10 +537,10 @@ def test_recognized_model_overrides_class_default(self): result = cls.get_default_configuration("tts") assert result.capabilities.output_modalities == frozenset({frozenset(["audio_path"])}) - def test_prompt_chat_target_preserves_system_prompt_for_recognized_model(self): - from pyrit.prompt_target.common.prompt_chat_target import PromptChatTarget + def test_prompt_target_preserves_system_prompt_for_recognized_model(self): + from pyrit.prompt_target.common.prompt_target import PromptTarget - result = PromptChatTarget.get_default_configuration("gpt-4o") + result = PromptTarget.get_default_configuration("gpt-4o") assert result.capabilities.supports_multi_turn is True assert result.capabilities.supports_multi_message_pieces is True diff --git a/tests/unit/prompt_target/target/test_target_requirements.py b/tests/unit/prompt_target/target/test_target_requirements.py index b2be2b7845..5c01395f4e 100644 --- a/tests/unit/prompt_target/target/test_target_requirements.py +++ b/tests/unit/prompt_target/target/test_target_requirements.py @@ -6,7 +6,7 @@ import pytest from pyrit.prompt_target import ( - CHAT_CONSUMER_REQUIREMENTS, + CHAT_TARGET_REQUIREMENTS, CapabilityName, TargetRequirements, ) @@ -35,8 +35,8 @@ def test_construction_from_frozenset(): assert reqs.required == {CapabilityName.MULTI_TURN, CapabilityName.JSON_OUTPUT} -def test_chat_consumer_requirements_shape(): - assert CHAT_CONSUMER_REQUIREMENTS.required == { +def test_chat_target_requirements_shape(): + assert CHAT_TARGET_REQUIREMENTS.required == { CapabilityName.EDITABLE_HISTORY, CapabilityName.MULTI_TURN, } @@ -58,7 +58,7 @@ def test_validate_passes_on_native_support(): ), ) - CHAT_CONSUMER_REQUIREMENTS.validate(target=target) + CHAT_TARGET_REQUIREMENTS.validate(target=target) def test_validate_passes_when_policy_is_adapt(): diff --git a/tests/unit/prompt_target/test_prompt_chat_target.py b/tests/unit/prompt_target/test_prompt_chat_target.py index 422196b4c6..cd4cf1a5d7 100644 --- a/tests/unit/prompt_target/test_prompt_chat_target.py +++ b/tests/unit/prompt_target/test_prompt_chat_target.py @@ -28,9 +28,7 @@ def test_is_response_format_json_false_when_no_metadata(): target = MockPromptTarget() piece = MagicMock(spec=MessagePiece) piece.prompt_metadata = None - # MockPromptTarget doesn't have is_response_format_json, use the base class method - result = PromptChatTarget.is_response_format_json(target, message_piece=piece) - assert result is False + assert target.is_response_format_json(message_piece=piece) is False @pytest.mark.usefixtures("patch_central_database") @@ -38,9 +36,9 @@ def test_is_response_format_json_true_when_json_format(): target = MockPromptTarget() piece = MagicMock(spec=MessagePiece) piece.prompt_metadata = {"response_format": "json"} - # PromptChatTarget default capabilities don't support json_output, so this should raise + # Default MockPromptTarget capabilities don't support json_output, so this should raise with pytest.raises(ValueError, match="does not support JSON response format"): - PromptChatTarget.is_response_format_json(target, message_piece=piece) + target.is_response_format_json(message_piece=piece) @pytest.mark.usefixtures("patch_central_database") @@ -50,14 +48,7 @@ def test_is_response_format_json_true_with_json_capable_target(): target._configuration = custom_conf piece = MagicMock(spec=MessagePiece) piece.prompt_metadata = {"response_format": "json"} - result = PromptChatTarget.is_response_format_json(target, message_piece=piece) - assert result is True - - -@pytest.mark.usefixtures("patch_central_database") -def test_default_configuration_class_attribute(): - assert PromptChatTarget._DEFAULT_CONFIGURATION.capabilities.supports_multi_turn is True - assert PromptChatTarget._DEFAULT_CONFIGURATION.capabilities.supports_system_prompt is True + assert target.is_response_format_json(message_piece=piece) is True @pytest.mark.usefixtures("patch_central_database") @@ -80,20 +71,108 @@ async def _send_prompt_to_target_async(self, *, normalized_conversation: list[Me return [] deprecation_warnings = [w for w in caught if issubclass(w.category, DeprecationWarning)] - assert len(deprecation_warnings) == 1 - assert "_DEFAULT_CAPABILITIES is deprecated" in str(deprecation_warnings[0].message) + assert any("_DEFAULT_CAPABILITIES is deprecated" in str(w.message) for w in deprecation_warnings) assert isinstance(_LegacyTarget._DEFAULT_CONFIGURATION, TargetConfiguration) assert _LegacyTarget._DEFAULT_CONFIGURATION.capabilities.supports_multi_turn is True @pytest.mark.usefixtures("patch_central_database") -def test_get_default_capabilities_emits_deprecation_warning(): +def test_subclassing_prompt_chat_target_emits_deprecation_warning(): with warnings.catch_warnings(record=True) as caught: warnings.simplefilter("always") - result = PromptChatTarget.get_default_capabilities(None) - deprecation_warnings = [w for w in caught if issubclass(w.category, DeprecationWarning)] - assert len(deprecation_warnings) == 1 - assert "get_default_capabilities() is deprecated" in str(deprecation_warnings[0].message) - assert isinstance(result, TargetCapabilities) - assert result.supports_multi_turn is True + class _LegacyChatSubclass(PromptChatTarget): + async def _send_prompt_to_target_async(self, *, normalized_conversation: list[Message]) -> list[Message]: + return [] + + deprecation_warnings = [ + w + for w in caught + if issubclass(w.category, DeprecationWarning) + and "PromptChatTarget" in str(w.message) + and "deprecated" in str(w.message) + ] + assert len(deprecation_warnings) >= 1 + + +@pytest.mark.usefixtures("patch_central_database") +def test_set_system_prompt_available_on_prompt_target(): + """The set_system_prompt API now lives on PromptTarget directly.""" + assert hasattr(PromptTarget, "set_system_prompt") + assert hasattr(PromptTarget, "is_response_format_json") + + +class _BarePromptTarget(PromptTarget): + """Minimal PromptTarget subclass that does not override set_system_prompt.""" + + async def _send_prompt_to_target_async(self, *, normalized_conversation: list[Message]) -> list[Message]: + return [] + + +@pytest.mark.usefixtures("patch_central_database") +@pytest.mark.parametrize( + "supports_multi_turn,supports_editable_history", + [ + (False, True), + (True, False), + (False, False), + ], +) +def test_set_system_prompt_raises_when_capabilities_missing(supports_multi_turn: bool, supports_editable_history: bool): + """set_system_prompt must require both multi-turn and editable-history capabilities.""" + config = TargetConfiguration( + capabilities=TargetCapabilities( + supports_multi_turn=supports_multi_turn, + supports_editable_history=supports_editable_history, + ) + ) + target = _BarePromptTarget(custom_configuration=config) + + with pytest.raises(ValueError, match="does not support setting a system prompt"): + target.set_system_prompt( + system_prompt="you are a helpful assistant", + conversation_id="conv-1", + ) + + +@pytest.mark.usefixtures("patch_central_database") +def test_set_system_prompt_writes_system_message_when_capabilities_present(): + """set_system_prompt writes a system-role message to memory on a capable target.""" + config = TargetConfiguration( + capabilities=TargetCapabilities( + supports_multi_turn=True, + supports_editable_history=True, + ) + ) + target = _BarePromptTarget(custom_configuration=config) + conversation_id = "conv-success" + + target.set_system_prompt( + system_prompt="you are a helpful assistant", + conversation_id=conversation_id, + ) + + messages = target._memory.get_conversation(conversation_id=conversation_id) + assert len(messages) == 1 + pieces = messages[0].message_pieces + assert len(pieces) == 1 + assert pieces[0].get_role_for_storage() == "system" + assert pieces[0].original_value == "you are a helpful assistant" + + +@pytest.mark.usefixtures("patch_central_database") +def test_set_system_prompt_raises_when_conversation_already_exists(): + """set_system_prompt must refuse to overwrite an existing conversation.""" + config = TargetConfiguration( + capabilities=TargetCapabilities( + supports_multi_turn=True, + supports_editable_history=True, + ) + ) + target = _BarePromptTarget(custom_configuration=config) + conversation_id = "conv-existing" + + target.set_system_prompt(system_prompt="first", conversation_id=conversation_id) + + with pytest.raises(RuntimeError, match="Conversation already exists"): + target.set_system_prompt(system_prompt="second", conversation_id=conversation_id) diff --git a/tests/unit/registry/test_target_registry.py b/tests/unit/registry/test_target_registry.py index 137c138f48..ce612f5203 100644 --- a/tests/unit/registry/test_target_registry.py +++ b/tests/unit/registry/test_target_registry.py @@ -7,7 +7,6 @@ from pyrit.identifiers import ComponentIdentifier from pyrit.models import Message, MessagePiece from pyrit.prompt_target import PromptTarget -from pyrit.prompt_target.common.prompt_chat_target import PromptChatTarget from pyrit.registry.object_registries.target_registry import TargetRegistry @@ -33,8 +32,8 @@ def _validate_request(self, *, normalized_conversation: list[Message]) -> None: pass -class MockPromptChatTarget(PromptChatTarget): - """Mock PromptChatTarget for testing conversation history support.""" +class MockPromptChatTarget(PromptTarget): + """Mock chat-style target for testing conversation history support.""" def __init__(self, *, model_name: str = "mock_chat_model", endpoint: str = "http://chat-test") -> None: super().__init__(model_name=model_name, endpoint=endpoint) diff --git a/tests/unit/scenario/test_cyber.py b/tests/unit/scenario/test_cyber.py index 1a1141b3c5..e5833525bf 100644 --- a/tests/unit/scenario/test_cyber.py +++ b/tests/unit/scenario/test_cyber.py @@ -11,7 +11,6 @@ from pyrit.identifiers import ComponentIdentifier from pyrit.models import SeedAttackGroup, SeedObjective, SeedPrompt from pyrit.prompt_target import PromptTarget -from pyrit.prompt_target.common.prompt_chat_target import PromptChatTarget from pyrit.registry.object_registries.attack_technique_registry import AttackTechniqueRegistry from pyrit.scenario.core.dataset_configuration import DatasetConfiguration from pyrit.scenario.core.scenario_techniques import register_scenario_techniques @@ -46,7 +45,7 @@ def mock_objective_target(): @pytest.fixture def mock_adversarial_target(): - mock = MagicMock(spec=PromptChatTarget) + mock = MagicMock(spec=PromptTarget) mock.get_identifier.return_value = _mock_id("MockAdversarialTarget") return mock diff --git a/tests/unit/scenario/test_foundry.py b/tests/unit/scenario/test_foundry.py index ab1995e947..e07f51e842 100644 --- a/tests/unit/scenario/test_foundry.py +++ b/tests/unit/scenario/test_foundry.py @@ -14,7 +14,6 @@ from pyrit.models import SeedAttackGroup, SeedObjective from pyrit.prompt_converter import Base64Converter from pyrit.prompt_target import PromptTarget -from pyrit.prompt_target.common.prompt_chat_target import PromptChatTarget from pyrit.scenario import AtomicAttack, DatasetConfiguration, ScenarioCompositeStrategy from pyrit.scenario.foundry import FoundryComposite, FoundryStrategy, RedTeamAgent from pyrit.score import FloatScaleThresholdScorer, TrueFalseScorer @@ -69,7 +68,7 @@ def mock_objective_target(): @pytest.fixture def mock_adversarial_target(): """Create a mock adversarial target for testing.""" - mock = MagicMock(spec=PromptChatTarget) + mock = MagicMock(spec=PromptTarget) mock.get_identifier.return_value = _mock_target_id("MockAdversarialTarget") return mock diff --git a/tests/unit/scenario/test_leakage_scenario.py b/tests/unit/scenario/test_leakage_scenario.py index d4b3d2664c..1be0984cc4 100644 --- a/tests/unit/scenario/test_leakage_scenario.py +++ b/tests/unit/scenario/test_leakage_scenario.py @@ -12,7 +12,7 @@ from pyrit.executor.attack.core.attack_config import AttackScoringConfig from pyrit.identifiers import ComponentIdentifier from pyrit.models import SeedAttackGroup, SeedDataset, SeedObjective -from pyrit.prompt_target import OpenAIChatTarget, PromptChatTarget, PromptTarget +from pyrit.prompt_target import OpenAIChatTarget, PromptTarget from pyrit.scenario import DatasetConfiguration from pyrit.scenario.airt import Leakage, LeakageStrategy from pyrit.score import TrueFalseCompositeScorer @@ -91,7 +91,7 @@ def mock_objective_scorer(): @pytest.fixture def mock_adversarial_target(): - mock = MagicMock(spec=PromptChatTarget) + mock = MagicMock(spec=PromptTarget) mock.get_identifier.return_value = _mock_target_id("MockAdversarialTarget") return mock diff --git a/tests/unit/scenario/test_psychosocial_harms.py b/tests/unit/scenario/test_psychosocial_harms.py index 666c4cb5f8..a8e3325765 100644 --- a/tests/unit/scenario/test_psychosocial_harms.py +++ b/tests/unit/scenario/test_psychosocial_harms.py @@ -10,7 +10,7 @@ from pyrit.common.path import DATASETS_PATH from pyrit.identifiers import ComponentIdentifier from pyrit.models import SeedAttackGroup, SeedDataset, SeedGroup, SeedObjective -from pyrit.prompt_target import OpenAIChatTarget, PromptChatTarget +from pyrit.prompt_target import OpenAIChatTarget, PromptTarget from pyrit.scenario.scenarios.airt import ( Psychosocial, PsychosocialStrategy, @@ -68,8 +68,8 @@ def mock_runtime_env(): @pytest.fixture -def mock_objective_target() -> PromptChatTarget: - mock = MagicMock(spec=PromptChatTarget) +def mock_objective_target() -> PromptTarget: + mock = MagicMock(spec=PromptTarget) mock.get_identifier.return_value = ComponentIdentifier(class_name="MockObjectiveTarget", class_module="test") return mock @@ -82,8 +82,8 @@ def mock_objective_scorer() -> FloatScaleThresholdScorer: @pytest.fixture -def mock_adversarial_target() -> PromptChatTarget: - mock = MagicMock(spec=PromptChatTarget) +def mock_adversarial_target() -> PromptTarget: + mock = MagicMock(spec=PromptTarget) mock.get_identifier.return_value = ComponentIdentifier(class_name="MockAdversarialTarget", class_module="test") return mock @@ -194,7 +194,7 @@ async def test_attack_generation_for_all( async def test_attack_runs_include_objectives_async( self, *, - mock_objective_target: PromptChatTarget, + mock_objective_target: PromptTarget, mock_objective_scorer: FloatScaleThresholdScorer, mock_resolved_seed_data, mock_dataset_config, @@ -214,7 +214,7 @@ async def test_attack_runs_include_objectives_async( async def test_get_atomic_attacks_async_returns_attacks( self, *, - mock_objective_target: PromptChatTarget, + mock_objective_target: PromptTarget, mock_objective_scorer: FloatScaleThresholdScorer, mock_resolved_seed_data, mock_dataset_config, @@ -238,7 +238,7 @@ class TestPsychosocialHarmsLifecycle: async def test_initialize_async_with_max_concurrency( self, *, - mock_objective_target: PromptChatTarget, + mock_objective_target: PromptTarget, mock_objective_scorer: FloatScaleThresholdScorer, mock_resolved_seed_data, mock_dataset_config, @@ -254,7 +254,7 @@ async def test_initialize_async_with_max_concurrency( async def test_initialize_async_with_memory_labels( self, *, - mock_objective_target: PromptChatTarget, + mock_objective_target: PromptTarget, mock_objective_scorer: FloatScaleThresholdScorer, mock_resolved_seed_data, mock_dataset_config, @@ -299,7 +299,7 @@ def test_get_default_strategy(self) -> None: async def test_no_target_duplication_async( self, *, - mock_objective_target: PromptChatTarget, + mock_objective_target: PromptTarget, mock_resolved_seed_data, mock_dataset_config, ) -> None: diff --git a/tests/unit/scenario/test_rapid_response.py b/tests/unit/scenario/test_rapid_response.py index ddf95df2e6..e830054b0b 100644 --- a/tests/unit/scenario/test_rapid_response.py +++ b/tests/unit/scenario/test_rapid_response.py @@ -18,7 +18,6 @@ from pyrit.identifiers import ComponentIdentifier from pyrit.models import SeedAttackGroup, SeedObjective, SeedPrompt from pyrit.prompt_target import OpenAIChatTarget, PromptTarget -from pyrit.prompt_target.common.prompt_chat_target import PromptChatTarget from pyrit.registry.object_registries.attack_technique_registry import AttackTechniqueRegistry, AttackTechniqueSpec from pyrit.scenario.core.attack_technique_factory import AttackTechniqueFactory from pyrit.scenario.core.dataset_configuration import DatasetConfiguration @@ -67,7 +66,7 @@ def mock_objective_target(): @pytest.fixture def mock_adversarial_target(): - mock = MagicMock(spec=PromptChatTarget) + mock = MagicMock(spec=PromptTarget) mock.get_identifier.return_value = _mock_id("MockAdversarialTarget") return mock diff --git a/tests/unit/scenario/test_scam.py b/tests/unit/scenario/test_scam.py index 95305fac3f..80092bb98b 100644 --- a/tests/unit/scenario/test_scam.py +++ b/tests/unit/scenario/test_scam.py @@ -17,7 +17,7 @@ from pyrit.executor.attack.core.attack_config import AttackScoringConfig from pyrit.identifiers import ComponentIdentifier from pyrit.models import SeedAttackGroup, SeedDataset, SeedGroup, SeedObjective -from pyrit.prompt_target import OpenAIChatTarget, PromptChatTarget, PromptTarget +from pyrit.prompt_target import OpenAIChatTarget, PromptTarget from pyrit.scenario import DatasetConfiguration from pyrit.scenario.scenarios.airt.scam import Scam, ScamStrategy from pyrit.score import TrueFalseCompositeScorer @@ -111,8 +111,8 @@ def mock_objective_scorer() -> TrueFalseCompositeScorer: @pytest.fixture -def mock_adversarial_target() -> PromptChatTarget: - mock = MagicMock(spec=PromptChatTarget) +def mock_adversarial_target() -> PromptTarget: + mock = MagicMock(spec=PromptTarget) mock.get_identifier.return_value = _mock_target_id("MockAdversarialTarget") return mock diff --git a/tests/unit/score/test_insecure_code_scorer.py b/tests/unit/score/test_insecure_code_scorer.py index fd5a022d9e..a300f50079 100644 --- a/tests/unit/score/test_insecure_code_scorer.py +++ b/tests/unit/score/test_insecure_code_scorer.py @@ -8,13 +8,13 @@ from pyrit.exceptions.exception_classes import InvalidJsonException from pyrit.identifiers import ComponentIdentifier from pyrit.models import MessagePiece, Score, UnvalidatedScore -from pyrit.prompt_target import PromptChatTarget +from pyrit.prompt_target import PromptTarget from pyrit.score import InsecureCodeScorer @pytest.fixture def mock_chat_target(patch_central_database): - return MagicMock(spec=PromptChatTarget) + return MagicMock(spec=PromptTarget) async def test_insecure_code_scorer_valid_response(mock_chat_target): diff --git a/tests/unit/score/test_scorer.py b/tests/unit/score/test_scorer.py index a35fbe3cb1..731a8068e3 100644 --- a/tests/unit/score/test_scorer.py +++ b/tests/unit/score/test_scorer.py @@ -13,7 +13,7 @@ from pyrit.identifiers import ComponentIdentifier from pyrit.memory import CentralMemory from pyrit.models import Message, MessagePiece, Score -from pyrit.prompt_target import PromptChatTarget +from pyrit.prompt_target import PromptTarget from pyrit.score import ( Scorer, ScorerPromptValidator, @@ -150,7 +150,7 @@ def get_scorer_metrics(self): @pytest.mark.parametrize("bad_json", [BAD_JSON, KEY_ERROR_JSON, KEY_ERROR2_JSON]) async def test_scorer_send_chat_target_async_bad_json_exception_retries(bad_json: str): - chat_target = MagicMock(PromptChatTarget) + chat_target = MagicMock(PromptTarget) chat_target.get_identifier.return_value = get_mock_target_identifier("MockChatTarget") bad_json_resp = Message( message_pieces=[MessagePiece(role="assistant", original_value=bad_json, conversation_id="test-convo")] @@ -173,7 +173,7 @@ async def test_scorer_send_chat_target_async_bad_json_exception_retries(bad_json async def test_scorer_score_value_with_llm_exception_display_prompt_id(): - chat_target = MagicMock(PromptChatTarget) + chat_target = MagicMock(PromptTarget) chat_target.get_identifier.return_value = get_mock_target_identifier("MockChatTarget") chat_target.send_prompt_async = AsyncMock(side_effect=Exception("Test exception")) @@ -197,7 +197,7 @@ async def test_scorer_score_value_with_llm_use_provided_attack_identifier(good_j message = Message( message_pieces=[MessagePiece(role="assistant", original_value=good_json, conversation_id="test-convo")] ) - chat_target = MagicMock(PromptChatTarget) + chat_target = MagicMock(PromptTarget) chat_target.get_identifier.return_value = get_mock_target_identifier("MockChatTarget") chat_target.send_prompt_async = AsyncMock(return_value=[message]) chat_target.set_system_prompt = MagicMock() @@ -231,7 +231,7 @@ async def test_scorer_score_value_with_llm_does_not_add_score_prompt_id_for_empt message = Message( message_pieces=[MessagePiece(role="assistant", original_value=good_json, conversation_id="test-convo")] ) - chat_target = MagicMock(PromptChatTarget) + chat_target = MagicMock(PromptTarget) chat_target.get_identifier.return_value = get_mock_target_identifier("MockChatTarget") chat_target.send_prompt_async = AsyncMock(return_value=[message]) chat_target.set_system_prompt = MagicMock() @@ -257,7 +257,7 @@ async def test_scorer_score_value_with_llm_does_not_add_score_prompt_id_for_empt async def test_scorer_send_chat_target_async_good_response(good_json): - chat_target = MagicMock(PromptChatTarget) + chat_target = MagicMock(PromptTarget) chat_target.get_identifier.return_value = get_mock_target_identifier("MockChatTarget") good_json_resp = Message( @@ -281,7 +281,7 @@ async def test_scorer_send_chat_target_async_good_response(good_json): async def test_scorer_remove_markdown_json_called(good_json): - chat_target = MagicMock(PromptChatTarget) + chat_target = MagicMock(PromptTarget) chat_target.get_identifier.return_value = get_mock_target_identifier("MockChatTarget") good_json_resp = Message( message_pieces=[MessagePiece(role="assistant", original_value=good_json, conversation_id="test-convo")] @@ -306,7 +306,7 @@ async def test_scorer_remove_markdown_json_called(good_json): async def test_score_value_with_llm_prepended_text_message_piece_creates_multipiece_message(good_json): """Test that prepended_text_message_piece creates a multi-piece message (text context + main content).""" - chat_target = MagicMock(PromptChatTarget) + chat_target = MagicMock(PromptTarget) chat_target.get_identifier.return_value = get_mock_target_identifier("MockChatTarget") good_json_resp = Message( message_pieces=[MessagePiece(role="assistant", original_value=good_json, conversation_id="test-convo")] @@ -349,7 +349,7 @@ async def test_score_value_with_llm_prepended_text_message_piece_creates_multipi async def test_score_value_with_llm_no_prepended_text_creates_single_piece_message(good_json): """Test that without prepended_text_message_piece, only a single piece message is created.""" - chat_target = MagicMock(PromptChatTarget) + chat_target = MagicMock(PromptTarget) chat_target.get_identifier.return_value = get_mock_target_identifier("MockChatTarget") good_json_resp = Message( message_pieces=[MessagePiece(role="assistant", original_value=good_json, conversation_id="test-convo")] @@ -384,7 +384,7 @@ async def test_score_value_with_llm_no_prepended_text_creates_single_piece_messa async def test_score_value_with_llm_prepended_text_works_with_audio(good_json): """Test that prepended_text_message_piece works with audio content (type-independent).""" - chat_target = MagicMock(PromptChatTarget) + chat_target = MagicMock(PromptTarget) chat_target.get_identifier.return_value = get_mock_target_identifier("MockChatTarget") good_json_resp = Message( message_pieces=[MessagePiece(role="assistant", original_value=good_json, conversation_id="test-convo")] @@ -1431,7 +1431,7 @@ async def test_blocked_takes_precedence_over_generic_error( async def test_score_value_with_llm_skips_reasoning_piece(good_json): """Test that _score_value_with_llm extracts JSON from the text piece, not a reasoning piece.""" - chat_target = MagicMock(PromptChatTarget) + chat_target = MagicMock(PromptTarget) chat_target.get_identifier.return_value = get_mock_target_identifier("MockChatTarget") # Simulate a reasoning model response: first piece is reasoning, second is the actual text with JSON diff --git a/tests/unit/setup/test_scenarios_initializer.py b/tests/unit/setup/test_scenarios_initializer.py index 4d9cfa841f..9e372f804f 100644 --- a/tests/unit/setup/test_scenarios_initializer.py +++ b/tests/unit/setup/test_scenarios_initializer.py @@ -11,7 +11,7 @@ from pyrit.common.path import EXECUTOR_SEED_PROMPT_PATH from pyrit.executor.attack import PromptSendingAttack from pyrit.models import SeedPrompt -from pyrit.prompt_target.common.prompt_chat_target import PromptChatTarget +from pyrit.prompt_target import PromptTarget from pyrit.registry import TargetRegistry from pyrit.registry.object_registries.attack_technique_registry import AttackTechniqueRegistry from pyrit.setup.initializers import ScenarioTechniqueInitializer @@ -41,7 +41,7 @@ def reset_registries(): @pytest.fixture def mock_adversarial_target(): """A mock adversarial target registered as 'adversarial_chat' so build_scenario_techniques resolves cleanly.""" - target = MagicMock(spec=PromptChatTarget) + target = MagicMock(spec=PromptTarget) # capabilities check inside get_default_adversarial_target requires multi_turn support target.capabilities.includes.return_value = True registry = TargetRegistry.get_registry_singleton() @@ -238,7 +238,7 @@ async def test_falls_back_to_default_target_when_registry_empty(self): # Patch OpenAIChatTarget at the import site inside scenario_techniques # (which is what get_default_adversarial_target calls), so the test does # not depend on OPENAI_CHAT_MODEL or any other env var being set. - fallback_target = MagicMock(spec=PromptChatTarget) + fallback_target = MagicMock(spec=PromptTarget) with patch( "pyrit.scenario.core.scenario_techniques.OpenAIChatTarget", return_value=fallback_target,