diff --git a/.github/instructions/scenarios.instructions.md b/.github/instructions/scenarios.instructions.md index 640ec9b544..dce9f3bbe0 100644 --- a/.github/instructions/scenarios.instructions.md +++ b/.github/instructions/scenarios.instructions.md @@ -14,7 +14,7 @@ All scenarios inherit from `Scenario` (ABC) and must: 1. **Define `VERSION`** as a class constant (increment on breaking changes) 2. **Optionally declare `BASELINE_ATTACK_POLICY`** (defaults to `BaselineAttackPolicy.Enabled` — a baseline `PromptSendingAttack` is prepended and callers can opt out per run by setting `include_baseline=False` in the run params, see "Run Parameters" below): - - `BaselineAttackPolicy.Disabled` — baseline supported but off by default (e.g. `Jailbreak`, where templates dominate the run). + - `BaselineAttackPolicy.Disabled` — baseline supported but off by default (e.g. `garak.doctor`, where the harm-probe technique set dominates the run). - `BaselineAttackPolicy.Forbidden` — baseline is meaningless for this scenario's comparison axis (e.g. `AdversarialBenchmark`, which compares against gold-standard answers). Supplying `include_baseline=True` raises `ValueError`. 3. **Pass `technique_class`, `default_technique`, and `default_dataset_config` to `super().__init__()`:** diff --git a/doc/code/scenarios/0_attack_techniques.ipynb b/doc/code/scenarios/0_attack_techniques.ipynb index 5f74c6f0a2..4680f8b891 100644 --- a/doc/code/scenarios/0_attack_techniques.ipynb +++ b/doc/code/scenarios/0_attack_techniques.ipynb @@ -59,7 +59,7 @@ " any scenario can use (the `role_play_*` variants, `many_shot`, `tap`, the `crescendo_*` variants, `red_teaming`,\n", " `context_compliance`). Registered by default.\n", "- [`extra.py`](../../../pyrit/setup/initializers/techniques/extra.py) — opt-in techniques that are\n", - " not part of the default set (`pair`, `violent_durian`).\n", + " not part of the default set (`pair`, `violent_durian`, `skeleton_key`).\n", "- [`airt.py`](../../../pyrit/setup/initializers/techniques/airt.py) — source-owned techniques that\n", " belong to a specific AIRT scenario. Unlike `core`/`extra`, these are imported directly by their\n", " owning scenario and are *not* part of the default aggregation.\n", @@ -112,6 +112,7 @@ " role_play_persuasion_written PromptSendingAttack yes single_turn, light, core\n", " role_play_trivia_game PromptSendingAttack yes single_turn, light, core\n", " role_play_video_game PromptSendingAttack yes single_turn, light, core\n", + " skeleton_key SkeletonKeyAttack no single_turn, extra\n", " tap TreeOfAttacksWithPruningAttack yes multi_turn, core\n", " violent_durian RedTeamingAttack yes multi_turn, extra\n" ] diff --git a/doc/code/scenarios/0_attack_techniques.py b/doc/code/scenarios/0_attack_techniques.py index 3c40b39532..0eaa5c97cb 100644 --- a/doc/code/scenarios/0_attack_techniques.py +++ b/doc/code/scenarios/0_attack_techniques.py @@ -58,7 +58,7 @@ # any scenario can use (the `role_play_*` variants, `many_shot`, `tap`, the `crescendo_*` variants, `red_teaming`, # `context_compliance`). Registered by default. # - [`extra.py`](../../../pyrit/setup/initializers/techniques/extra.py) — opt-in techniques that are -# not part of the default set (`pair`, `violent_durian`). +# not part of the default set (`pair`, `violent_durian`, `skeleton_key`). # - [`airt.py`](../../../pyrit/setup/initializers/techniques/airt.py) — source-owned techniques that # belong to a specific AIRT scenario. Unlike `core`/`extra`, these are imported directly by their # owning scenario and are *not* part of the default aggregation. diff --git a/doc/scanner/airt.ipynb b/doc/scanner/airt.ipynb index b6198e0f63..8f6456c2ef 100644 --- a/doc/scanner/airt.ipynb +++ b/doc/scanner/airt.ipynb @@ -526,18 +526,28 @@ "source": [ "## Jailbreak\n", "\n", - "Tests target resilience against template-based jailbreak attacks using various prompt injection\n", - "templates.\n", + "Tests target resilience against jailbreak templates. A run crosses three selectors: the harmful\n", + "objectives (**dataset**, HarmBench), the **techniques** each jailbreak is delivered through, and\n", + "which **jailbreaks** to run. Two deliveries are on by default: `prompt_sending` renders the\n", + "objective inline into the template as a request converter (target-agnostic), and\n", + "`jailbreak_system_prompt` sets the template as a native system prompt with the objective sent as\n", + "the user turn (only for targets that natively support editable history + system prompts — it is\n", + "skipped for incapable targets). Registry techniques like `role_play_*`, `many_shot`, and `tap` are\n", + "opt-in. Results are grouped by jailbreak template, and a baseline (the un-jailbroken objective) is\n", + "included by default so complying with the bare objective is itself visible.\n", "\n", "```bash\n", "pyrit_scan airt.jailbreak \\\n", - " --initializers target \\\n", + " --initializers target load_default_datasets \\\n", " --target openai_chat \\\n", - " --techniques prompt_sending \\\n", + " --dataset-names harmbench \\\n", " --max-dataset-size 1\n", "```\n", "\n", - "**Available techniques:** ALL, SIMPLE, COMPLEX, PromptSending, ManyShot, SkeletonKey, RolePlay" + "**Available techniques:** ALL, DEFAULT (`prompt_sending` + `jailbreak_system_prompt`), plus registry\n", + "techniques (`role_play_*`, `many_shot`, `tap`, …). By default a small random sample of jailbreak\n", + "templates runs; pass `num_jailbreaks` (random count) or `jailbreak_names` (explicit) to widen or\n", + "pin the selection." ] }, { @@ -564,13 +574,14 @@ "source": [ "from pyrit.scenario.airt import Jailbreak, JailbreakTechnique\n", "\n", - "dataset_config = DatasetAttackConfiguration(dataset_names=[\"airt_harms\"], max_dataset_size=1)\n", + "dataset_config = DatasetAttackConfiguration(dataset_names=[\"harmbench\"], max_dataset_size=1)\n", "\n", "scenario = Jailbreak()\n", "scenario.set_params_from_args( # type: ignore\n", " args={\n", " \"objective_target\": objective_target,\n", - " \"scenario_techniques\": [JailbreakTechnique.PromptSending],\n", + " \"scenario_techniques\": [JailbreakTechnique.DEFAULT],\n", + " \"jailbreak_names\": [\"aim.yaml\"],\n", " \"dataset_config\": dataset_config,\n", " }\n", ")\n", diff --git a/doc/scanner/airt.py b/doc/scanner/airt.py index 9b3c513657..c9003c6bba 100644 --- a/doc/scanner/airt.py +++ b/doc/scanner/airt.py @@ -165,29 +165,40 @@ # %% [markdown] # ## Jailbreak # -# Tests target resilience against template-based jailbreak attacks using various prompt injection -# templates. +# Tests target resilience against jailbreak templates. A run crosses three selectors: the harmful +# objectives (**dataset**, HarmBench), the **techniques** each jailbreak is delivered through, and +# which **jailbreaks** to run. Two deliveries are on by default: `prompt_sending` renders the +# objective inline into the template as a request converter (target-agnostic), and +# `jailbreak_system_prompt` sets the template as a native system prompt with the objective sent as +# the user turn (only for targets that natively support editable history + system prompts — it is +# skipped for incapable targets). Registry techniques like `role_play_*`, `many_shot`, and `tap` are +# opt-in. Results are grouped by jailbreak template, and a baseline (the un-jailbroken objective) is +# included by default so complying with the bare objective is itself visible. # # ```bash # pyrit_scan airt.jailbreak \ -# --initializers target \ +# --initializers target load_default_datasets \ # --target openai_chat \ -# --techniques prompt_sending \ +# --dataset-names harmbench \ # --max-dataset-size 1 # ``` # -# **Available techniques:** ALL, SIMPLE, COMPLEX, PromptSending, ManyShot, SkeletonKey, RolePlay +# **Available techniques:** ALL, DEFAULT (`prompt_sending` + `jailbreak_system_prompt`), plus registry +# techniques (`role_play_*`, `many_shot`, `tap`, …). By default a small random sample of jailbreak +# templates runs; pass `num_jailbreaks` (random count) or `jailbreak_names` (explicit) to widen or +# pin the selection. # %% from pyrit.scenario.airt import Jailbreak, JailbreakTechnique -dataset_config = DatasetAttackConfiguration(dataset_names=["airt_harms"], max_dataset_size=1) +dataset_config = DatasetAttackConfiguration(dataset_names=["harmbench"], max_dataset_size=1) scenario = Jailbreak() scenario.set_params_from_args( # type: ignore args={ "objective_target": objective_target, - "scenario_techniques": [JailbreakTechnique.PromptSending], + "scenario_techniques": [JailbreakTechnique.DEFAULT], + "jailbreak_names": ["aim.yaml"], "dataset_config": dataset_config, } ) diff --git a/pyrit/scenario/scenarios/airt/__init__.py b/pyrit/scenario/scenarios/airt/__init__.py index a7f7a41d3f..61fdbdfb56 100644 --- a/pyrit/scenario/scenarios/airt/__init__.py +++ b/pyrit/scenario/scenarios/airt/__init__.py @@ -6,7 +6,7 @@ from typing import Any from pyrit.scenario.scenarios.airt.cyber import Cyber, _build_cyber_technique -from pyrit.scenario.scenarios.airt.jailbreak import Jailbreak, JailbreakTechnique +from pyrit.scenario.scenarios.airt.jailbreak import Jailbreak, _build_jailbreak_technique from pyrit.scenario.scenarios.airt.leakage import Leakage, _build_leakage_technique from pyrit.scenario.scenarios.airt.psychosocial import Psychosocial, PsychosocialTechnique from pyrit.scenario.scenarios.airt.rapid_response import RapidResponse, _build_rapid_response_technique @@ -29,6 +29,8 @@ def __getattr__(name: str) -> Any: return _build_leakage_technique() if name == "CyberTechnique": return _build_cyber_technique() + if name == "JailbreakTechnique": + return _build_jailbreak_technique() raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/pyrit/scenario/scenarios/airt/jailbreak.py b/pyrit/scenario/scenarios/airt/jailbreak.py index 16157f0316..8eb0857e72 100644 --- a/pyrit/scenario/scenarios/airt/jailbreak.py +++ b/pyrit/scenario/scenarios/airt/jailbreak.py @@ -1,85 +1,208 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT license. +from __future__ import annotations + +import logging +from functools import cache from pathlib import Path -from typing import Any +from typing import TYPE_CHECKING, Any, ClassVar from pyrit.common import apply_defaults -from pyrit.common.path import EXECUTOR_RED_TEAM_PATH, EXECUTOR_SIMULATED_TARGET_PATH from pyrit.converter import TextJailbreakConverter from pyrit.datasets import TextJailBreak -from pyrit.executor.attack.core.attack_config import AttackConverterConfig, AttackScoringConfig -from pyrit.executor.attack.single_turn.many_shot_jailbreak import ManyShotJailbreakAttack from pyrit.executor.attack.single_turn.prompt_sending import PromptSendingAttack -from pyrit.executor.attack.single_turn.skeleton_key import SkeletonKeyAttack -from pyrit.models import AttackSeedGroup -from pyrit.prompt_normalizer import ConverterConfiguration -from pyrit.prompt_target.common.prompt_target import PromptTarget -from pyrit.scenario.core.atomic_attack import AtomicAttack -from pyrit.scenario.core.attack_technique import AttackTechnique +from pyrit.models import AttackTechniqueSeedGroup, Parameter, SeedPrompt +from pyrit.prompt_target import CapabilityName +from pyrit.registry.components.attack_technique_registry import AttackTechniqueRegistry +from pyrit.registry.tag_query import TagQuery from pyrit.scenario.core.attack_technique_factory import AttackTechniqueFactory from pyrit.scenario.core.dataset_configuration import DatasetAttackConfiguration -from pyrit.scenario.core.scenario import Scenario -from pyrit.scenario.core.scenario_context import ScenarioContext -from pyrit.scenario.core.scenario_target_defaults import get_default_adversarial_target -from pyrit.scenario.core.scenario_technique import ScenarioTechnique -from pyrit.score import TrueFalseScorer +from pyrit.scenario.core.matrix_atomic_attack_builder import ( + MatrixAtomicAttackBuilder, + resolve_technique_factories, +) +from pyrit.scenario.core.scenario import BaselineAttackPolicy, Scenario + +if TYPE_CHECKING: + from collections.abc import Mapping + + from pyrit.converter import Converter + from pyrit.models import AttackSeedGroup + from pyrit.prompt_target import PromptTarget + from pyrit.scenario.core.atomic_attack import AtomicAttack + from pyrit.scenario.core.scenario_context import ScenarioContext + from pyrit.scenario.core.scenario_technique import ScenarioTechnique + from pyrit.score import TrueFalseScorer + +logger = logging.getLogger(__name__) + +# Metadata key under which the resolved jailbreak templates are persisted, so a resumed run +# replays the exact same set even when a random sample was drawn. +_JAILBREAK_TEMPLATES_METADATA_KEY = "jailbreak_templates" + +# How many jailbreak templates a bare run draws at random. Kept small so the default run stays fast +# — jailbreak templates multiply against objectives and techniques. Override per run with +# ``num_jailbreaks`` (random count) or ``jailbreak_names`` (an explicit set). +_DEFAULT_NUM_JAILBREAKS = 2 + +# Scenario-local default techniques. Both "just send" the jailbreak; they differ only in *where* the +# framing lands: +# - ``prompt_sending`` renders the template inline into the user message (target-agnostic). +# - ``jailbreak_system_prompt`` sets the template as the system prompt and sends the objective as +# the user turn (only for targets that natively support editable history + system prompts). +# The registry deliberately omits a bare send, and system-prompt delivery is scenario-specific, so +# both are injected locally (like Leakage's ``first_letter`` / ``image``) and form the default set. +_PROMPT_SENDING = "prompt_sending" +_JAILBREAK_SYSTEM_PROMPT = "jailbreak_system_prompt" +_DEFAULT_TECHNIQUES: frozenset[str] = frozenset({_PROMPT_SENDING, _JAILBREAK_SYSTEM_PROMPT}) + + +@cache +def _prompt_sending_factory() -> AttackTechniqueFactory: + """ + Build the scenario-local ``prompt_sending`` ("just send") technique factory. + + Returns: + AttackTechniqueFactory: A ``PromptSendingAttack`` factory with no seed technique, so the + objective (jailbroken inline by the ``TextJailbreakConverter``) is sent directly with no + additional attack layered on top. + """ + return AttackTechniqueFactory( + name=_PROMPT_SENDING, + attack_class=PromptSendingAttack, + technique_tags=["single_turn"], + ) -class JailbreakTechnique(ScenarioTechnique): +@cache +def _jailbreak_system_prompt_factory() -> AttackTechniqueFactory: """ - Technique for jailbreak attacks. + Build the scenario-local ``jailbreak_system_prompt`` delivery technique factory. - The SIMPLE technique just sends the jailbroken prompt and records the response. It is meant to - expose an obvious way of using this scenario without worrying about additional tweaks and changes - to the prompt. + This base factory only carries the technique name and tags so the technique appears in the enum + and the default set. The per-run system framing (which template to deliver) is attached in + ``_build_atomic_attacks_async`` via ``_build_system_prompt_factory``, because the framing text is + drawn per run. - COMPLEX techniques use additional techniques to enhance the jailbreak like modifying the - system prompt or probing the target model for an additional vulnerability (e.g. the SkeletonKeyAttack). - They are meant to provide a sense of how well a jailbreak generalizes to slight changes in the delivery - method. + Returns: + AttackTechniqueFactory: A ``PromptSendingAttack`` placeholder factory used for enum + construction and technique selection. """ + return AttackTechniqueFactory( + name=_JAILBREAK_SYSTEM_PROMPT, + attack_class=PromptSendingAttack, + technique_tags=["single_turn"], + ) - # Aggregate members (special markers that expand to techniques with matching tags) - ALL = ("all", {"all"}) - SIMPLE = ("simple", {"simple"}) - COMPLEX = ("complex", {"complex"}) - # Simple techniques - PromptSending = ("prompt_sending", {"simple"}) +def _extra_default_factories() -> dict[str, AttackTechniqueFactory]: + """Return the scenario-local default technique factories keyed by name.""" + return { + _PROMPT_SENDING: _prompt_sending_factory(), + _JAILBREAK_SYSTEM_PROMPT: _jailbreak_system_prompt_factory(), + } - # Complex techniques - ManyShot = ("many_shot", {"complex"}) - SkeletonKey = ("skeleton", {"complex"}) - RolePlay = ("role_play_persuasion", {"complex"}) - @classmethod - def get_aggregate_tags(cls) -> set[str]: - """ - Get the set of tags that represent aggregate categories. +@cache +def _build_jailbreak_technique() -> type[ScenarioTechnique]: + """ + Build the Jailbreak technique class dynamically from every registered factory plus the + scenario-local defaults. - Returns: - set[str]: Set of tags that are aggregate markers. - """ - # Include base class aggregates ("all") and add scenario-specific ones - return super().get_aggregate_tags() | {"simple", "complex"} + The technique axis is the set of *attack techniques* a jailbreak is delivered through: the two + default deliveries (``prompt_sending`` and ``jailbreak_system_prompt``) plus whatever techniques + are registered (``role_play_*``, ``many_shot``, ``tap``, …). Jailbreak templates are a separate + selector (``num_jailbreaks`` / ``jailbreak_names``), so only the two deliveries are on by default + — crossing every template with every registered technique explodes quickly. + + Returns: + type[ScenarioTechnique]: The dynamically generated technique enum class. + """ + registry = AttackTechniqueRegistry.get_registry_singleton() + factories = list(registry.get_factories_or_raise().values()) + list(_extra_default_factories().values()) + return AttackTechniqueRegistry.build_technique_class_from_factories( # type: ignore[return-value, ty:invalid-return-type] + class_name="JailbreakTechnique", + factories=factories, + aggregate_tags={ + "single_turn": TagQuery.any_of("single_turn"), + "multi_turn": TagQuery.any_of("multi_turn"), + }, + default_technique_names=set(_DEFAULT_TECHNIQUES), + ) class Jailbreak(Scenario): """ Jailbreak scenario implementation for PyRIT. - This scenario tests how vulnerable models are to jailbreak attacks by applying - various single-turn jailbreak templates to a set of test prompts. The responses are - scored to determine if the jailbreak was successful. + Tests how vulnerable a model is to jailbreak templates. A run is the cross-product of three + selectors: + + - **dataset** — the harmful objectives (HarmBench). + - **techniques** — the *attack techniques* each jailbreak is delivered through. Two deliveries + are on by default: ``prompt_sending`` (the template rendered inline into the user message) and + ``jailbreak_system_prompt`` (the template set as the system prompt with the objective sent as + the user turn). The registry techniques (``role_play_*``, ``many_shot``, ``tap``, …) are + opt-in. + - **jailbreaks** — which jailbreak templates to run (a random ``num_jailbreaks`` sample or an + explicit ``jailbreak_names`` set). + + ``prompt_sending`` applies each template as a ``TextJailbreakConverter`` on the outgoing request, + so the objective is rendered inline into the template's ``{{prompt}}`` slot; this keeps that + delivery target-agnostic and lets it compose with every technique. ``jailbreak_system_prompt`` + instead sets the template as a native system prompt and sends the objective as its own user turn, + so it is only built for targets that natively support editable history and system prompts (it is + skipped for incapable targets, or raises if it is the only selected technique). Responses are + scored to determine whether the jailbreak succeeded (non-refusal). """ - VERSION: int = 1 + VERSION: int = 3 + + #: Baseline (an un-jailbroken prompt-send over the objectives) is included by default: a model + #: that complies with the bare objective is itself interesting signal. Callers opt out per run + #: with ``include_baseline=False``. + BASELINE_ATTACK_POLICY: ClassVar[BaselineAttackPolicy] = BaselineAttackPolicy.Enabled @classmethod def required_datasets(cls) -> list[str]: """Return a list of dataset names required by this scenario.""" - return ["airt_harms"] + return ["harmbench"] + + @classmethod + def additional_parameters(cls) -> list[Parameter]: + """ + Declare the run-configurable parameters this scenario accepts (CLI / config file). + + Returns: + list[Parameter]: The jailbreak-template selectors (``num_jailbreaks``, ``jailbreak_names``) + and the ``num_jailbreak_attempts`` repeat-count parameter. + """ + return [ + Parameter( + name="num_jailbreaks", + description=( + "Draw this many random jailbreak templates for the run. Mutually exclusive with jailbreak_names." + ), + param_type=int, + default=None, + ), + Parameter( + name="num_jailbreak_attempts", + description="Number of times to try each (technique x jailbreak template x objective).", + param_type=int, + default=1, + ), + Parameter( + name="jailbreak_names", + description=( + "Explicit jailbreak template file names to run (e.g. aim.yaml dan_11.yaml). " + "When omitted, a random sample is drawn. Mutually exclusive with num_jailbreaks." + ), + param_type=list[str], + default=None, + ), + ] @apply_defaults def __init__( @@ -87,9 +210,6 @@ def __init__( *, objective_scorer: TrueFalseScorer | None = None, scenario_result_id: str | None = None, - num_templates: int | None = None, - num_attempts: int = 1, - jailbreak_names: list[str] | None = None, ) -> None: """ Initialize the jailbreak scenario. @@ -98,183 +218,263 @@ def __init__( objective_scorer (TrueFalseScorer | None): Scorer for detecting successful jailbreaks (non-refusal). If not provided, defaults to an inverted refusal scorer. scenario_result_id (str | None): Optional ID of an existing scenario result to resume. - num_templates (int | None): Choose num_templates random jailbreaks rather than using all of them. - num_attempts (int | None): Number of times to try each jailbreak. - jailbreak_names (list[str] | None): List of jailbreak names from the template list under datasets. - to use. - - Raises: - ValueError: If both jailbreak_names and num_templates are provided, as random selection - is incompatible with a predetermined list. - ValueError: If the jailbreak_names list contains a jailbreak that isn't in the listed - templates. - """ - if jailbreak_names is None: - jailbreak_names = [] - if jailbreak_names and num_templates: - raise ValueError( - "Please provide only one of `num_templates` (random selection)" - " or `jailbreak_names` (specific selection)." - ) - self._objective_scorer: TrueFalseScorer = ( objective_scorer if objective_scorer else self._get_default_objective_scorer() ) + # Resolved lazily at build time (from the run parameter bag) and cached so the same + # template set feeds both attack construction and the persisted metadata. + self._resolved_jailbreaks: list[str] = [] - self._num_templates = num_templates - self._num_attempts = num_attempts - self._adversarial_target: PromptTarget | None = None - - # Note that num_templates and jailbreak_names are mutually exclusive. - # If self._num_templates is None, then this returns all discoverable jailbreak templates. - # If self._num_templates has some value, then all_templates is a subset of all available - # templates, but jailbreak_names is guaranteed to be [], so diff = {}. - all_templates = TextJailBreak.get_jailbreak_templates(num_templates=self._num_templates) - - # Example: if jailbreak_names is {'a', 'b', 'c'}, and all_templates is {'b', 'c', 'd'}, - # then diff = {'a'}, which raises the error as 'a' was not discovered in all_templates. - diff = set(jailbreak_names) - set(all_templates) - if len(diff) > 0: - raise ValueError(f"Error: could not find templates `{diff}`!") - - # If jailbreak_names has some value, then `if jailbreak_names` passes, and self._jailbreaks - # is set to jailbreak_names. Otherwise we use all_templates. - self._jailbreaks = jailbreak_names if jailbreak_names else all_templates + technique_class = _build_jailbreak_technique() super().__init__( version=self.VERSION, - technique_class=JailbreakTechnique, - default_technique=JailbreakTechnique.SIMPLE, - default_dataset_config=DatasetAttackConfiguration(dataset_names=["airt_harms"], max_dataset_size=4), + technique_class=technique_class, + default_technique=technique_class("default"), + default_dataset_config=DatasetAttackConfiguration(dataset_names=["harmbench"], max_dataset_size=4), objective_scorer=self._objective_scorer, scenario_result_id=scenario_result_id, ) - def _get_or_create_adversarial_target(self) -> PromptTarget: + def _resolve_templates(self) -> list[str]: + """ + Resolve the jailbreak templates for this run, replaying the persisted set on resume. + + On a fresh run this reads the run parameters: an explicit ``jailbreak_names`` set or a random + ``num_jailbreaks`` sample (defaulting to a small random draw when neither is given). On resume + the originally chosen set is read back from the stored ``ScenarioResult`` metadata so a random + sample isn't redrawn (which would diverge from the persisted attacks). + + Returns: + list[str]: The jailbreak template file names to run. + + Raises: + ValueError: If both ``num_jailbreaks`` and ``jailbreak_names`` are provided, or if + ``jailbreak_names`` contains an unknown template. """ - Return the shared adversarial target, creating it on first access. + if self._scenario_result_id is not None: + stored = self._memory.get_scenario_results(scenario_result_ids=[self._scenario_result_id]) + if stored: + persisted = (stored[0].metadata or {}).get(_JAILBREAK_TEMPLATES_METADATA_KEY) + if persisted: + return list(persisted) - Reuses a single PromptTarget instance across all role-play attacks - to avoid repeated client and TLS setup. + num_jailbreaks = self.params.get("num_jailbreaks") + jailbreak_names = self.params.get("jailbreak_names") + + if jailbreak_names and num_jailbreaks: + raise ValueError( + "Please provide only one of `num_jailbreaks` (random selection)" + " or `jailbreak_names` (specific selection)." + ) + if jailbreak_names: + available = set(TextJailBreak.get_jailbreak_templates()) + diff = set(jailbreak_names) - available + if diff: + raise ValueError(f"Error: could not find templates `{diff}`!") + return list(jailbreak_names) + return TextJailBreak.get_jailbreak_templates(num_templates=num_jailbreaks or _DEFAULT_NUM_JAILBREAKS) + + def _build_initial_scenario_metadata(self) -> dict[str, Any]: + """ + Persist the resolved jailbreak templates alongside the base scenario metadata. Returns: - PromptTarget: The shared adversarial target. + dict[str, Any]: The base metadata plus the resolved jailbreak template set. """ - if self._adversarial_target is None: - self._adversarial_target = get_default_adversarial_target() - return self._adversarial_target + metadata = super()._build_initial_scenario_metadata() + metadata[_JAILBREAK_TEMPLATES_METADATA_KEY] = list(self._resolved_jailbreaks) + return metadata - async def _get_atomic_attack_from_technique_async( - self, *, technique: str, jailbreak_template_name: str, seed_groups: list[AttackSeedGroup] - ) -> AtomicAttack: + async def _build_atomic_attacks_async(self, *, context: ScenarioContext) -> list[AtomicAttack]: """ - Create an atomic attack for a specific jailbreak template. + Build one atomic attack per (technique x jailbreak template x dataset x attempt). + + ``prompt_sending`` (and any opt-in registry techniques) deliver each jailbreak template as a + ``TextJailbreakConverter`` appended to that technique's request converters, so the objective + is rendered inline into the template's ``{{prompt}}`` slot on the wire — target-agnostic and + composable with every technique. ``jailbreak_system_prompt`` instead delivers the template as + a native system prompt (no converter) with the objective sent as its own user turn, so it is + only built when the objective target natively supports editable history and system prompts. + Results group by jailbreak template so per-template ASR rolls up naturally. Args: - technique (str): JailbreakTechnique to use. - jailbreak_template_name (str): Name of the jailbreak template file. - seed_groups (list[AttackSeedGroup]): Seed groups the attack draws from. + context (ScenarioContext): The resolved runtime inputs for this run. Returns: - AtomicAttack: An atomic attack using the specified jailbreak template. + list[AtomicAttack]: The atomic attacks to execute. Raises: - ValueError: If scenario is not properly initialized. + ValueError: If the scenario is not properly initialized, or if + ``jailbreak_system_prompt`` is the only selected technique but the target cannot + support it. """ - # objective_target is guaranteed to be non-None by parent class validation if self._objective_target is None: raise ValueError( "Scenario not properly initialized. Call await scenario.initialize_async() before running." ) - # Create the jailbreak converter - jailbreak_converter = TextJailbreakConverter( - jailbreak_template=TextJailBreak(template_file_name=jailbreak_template_name) - ) + self._resolved_jailbreaks = self._resolve_templates() + num_attempts = self.params.get("num_jailbreak_attempts", 1) - # Create converter configuration - converter_config = AttackConverterConfig( - request_converters=ConverterConfiguration.from_converters(converters=[jailbreak_converter]) - ) + technique_factories = resolve_technique_factories(context=context, extra_factories=_extra_default_factories()) - attack: ManyShotJailbreakAttack | PromptSendingAttack | SkeletonKeyAttack | None = None - args: dict[str, Any] = { - "objective_target": self._objective_target, - "attack_scoring_config": AttackScoringConfig(objective_scorer=self._objective_scorer), - "attack_converter_config": converter_config, + # ``jailbreak_system_prompt`` is delivered separately (native system prompt, no converter); + # every other technique goes through the inline converter path. + system_selected = _JAILBREAK_SYSTEM_PROMPT in technique_factories + converter_factories = { + name: factory for name, factory in technique_factories.items() if name != _JAILBREAK_SYSTEM_PROMPT } - # Extract template name without extension for the atomic attack name - template_name = Path(jailbreak_template_name).stem - - match technique: - case "many_shot": - attack = ManyShotJailbreakAttack(**args) - case "prompt_sending": - attack = PromptSendingAttack(**args) - case "skeleton": - attack = SkeletonKeyAttack(**args) - case "role_play_persuasion": - # Role play is a simulated-conversation technique: an adversarial - # chat improvises a short persuasion role play, then the objective - # is delivered to the target with the jailbreak converter applied. - adversarial_target = self._get_or_create_adversarial_target() - role_play_technique = AttackTechniqueFactory.with_simulated_conversation( - name="role_play_persuasion", - adversarial_chat_system_prompt_path=EXECUTOR_RED_TEAM_PATH - / "role_play" - / "role_play_persuasion.yaml", - next_message_system_prompt_path=EXECUTOR_SIMULATED_TARGET_PATH / "role_play_next_message.yaml", - num_turns=2, - ).create( - objective_target=self._objective_target, - attack_scoring_config=AttackScoringConfig(objective_scorer=self._objective_scorer), - adversarial_chat=adversarial_target, - extra_request_converters=ConverterConfiguration.from_converters(converters=[jailbreak_converter]), + build_system_delivery = system_selected and self._target_supports_system_delivery(self._objective_target) + if system_selected and not build_system_delivery: + if not converter_factories: + raise ValueError( + "The 'jailbreak_system_prompt' technique needs a target that natively supports " + "editable history and system prompts. Choose a capable target or a different technique." ) - return AtomicAttack( - atomic_attack_name=f"jailbreak_{template_name}", - attack_technique=role_play_technique, - seed_groups=seed_groups, - adversarial_chat=adversarial_target, - objective_scorer=self._objective_scorer, + logger.warning( + "Skipping 'jailbreak_system_prompt' delivery: target does not natively support " + "editable history and system prompts. Running the remaining techniques." + ) + + builder = MatrixAtomicAttackBuilder( + objective_target=self._objective_target, + objective_scorer=self._objective_scorer, + memory_labels=context.memory_labels, + ) + + atomic_attacks: list[AtomicAttack] = [] + for template_file_name in self._resolved_jailbreaks: + template_stem = Path(template_file_name).stem + + if converter_factories: + jailbreak_converter = TextJailbreakConverter( + jailbreak_template=TextJailBreak(template_file_name=template_file_name) + ) + # Within the extra-converter stack, apply the jailbreak first (wrap the raw objective + # in the template), then any per-technique converters the caller layered on via + # ``--techniques :converter.*``. (A technique's own built-in converters, if any, + # still run ahead of this extra stack inside the factory.) + technique_converters = { + technique_name: [jailbreak_converter, *self._technique_converters.get(technique_name, [])] + for technique_name in converter_factories + } + atomic_attacks.extend( + self._build_delivery_attacks( + builder=builder, + technique_factories=converter_factories, + technique_converters=technique_converters, + dataset_groups=context.seed_groups_by_dataset, + template_stem=template_stem, + num_attempts=num_attempts, + ) ) - case _: - raise ValueError(f"Unknown JailbreakTechnique `{technique}`.") - if not attack: - raise ValueError(f"Attack cannot be None!") + if build_system_delivery: + system_factory = self._build_system_prompt_factory(template_file_name=template_file_name) + atomic_attacks.extend( + self._build_delivery_attacks( + builder=builder, + technique_factories={_JAILBREAK_SYSTEM_PROMPT: system_factory}, + technique_converters={}, + dataset_groups=context.seed_groups_by_dataset, + template_stem=template_stem, + num_attempts=num_attempts, + ) + ) + return atomic_attacks - return AtomicAttack( - atomic_attack_name=f"jailbreak_{template_name}", - attack_technique=AttackTechnique(attack=attack), - seed_groups=seed_groups, - ) + def _build_delivery_attacks( + self, + *, + builder: MatrixAtomicAttackBuilder, + technique_factories: dict[str, AttackTechniqueFactory], + technique_converters: dict[str, list[Converter]], + dataset_groups: Mapping[str, list[AttackSeedGroup]], + template_stem: str, + num_attempts: int, + ) -> list[AtomicAttack]: + """ + Build the (technique x dataset x attempt) atomic attacks for a single jailbreak template. - async def _build_atomic_attacks_async(self, *, context: ScenarioContext) -> list[AtomicAttack]: + Args: + builder (MatrixAtomicAttackBuilder): The shared matrix builder. + technique_factories (dict[str, AttackTechniqueFactory]): Factories to build for this + delivery path (either the converter techniques or the system-prompt technique). + technique_converters (dict[str, list[Converter]]): Per-technique extra request converters + (empty for the system-prompt delivery so the jailbreak is not double-applied). + dataset_groups (Mapping[str, list[AttackSeedGroup]]): Seed groups keyed by dataset name. + template_stem (str): The jailbreak template stem, used for naming and grouping. + num_attempts (int): How many times to repeat each combination. + + Returns: + list[AtomicAttack]: The atomic attacks for this template and delivery path. + """ + atomic_attacks: list[AtomicAttack] = [] + for attempt in range(num_attempts): + suffix = f"_attempt{attempt + 1}" if num_attempts > 1 else "" + atomic_attacks.extend( + builder.build( + technique_factories=technique_factories, + dataset_groups=dataset_groups, + technique_converters=technique_converters, + name_fn=lambda combo, stem=template_stem, suffix=suffix: ( + f"{combo.technique_name}_{stem}_{combo.dataset_name}{suffix}" + ), + display_group_fn=lambda combo, stem=template_stem: stem, + include_baseline=False, + ) + ) + return atomic_attacks + + def _build_system_prompt_factory(self, *, template_file_name: str) -> AttackTechniqueFactory: """ - Generate atomic attacks for each jailbreak template. + Build a ``jailbreak_system_prompt`` factory carrying a single template's framing. - This method creates an atomic attack for each retrieved jailbreak template. + The template is rendered with an empty prompt so it is pure framing (persona setup) and + attached as a ``role="system"`` technique seed. The objective is delivered separately as its + own user turn when the matrix builder merges this technique into each objective seed group, + so the framing never overwrites the objective. Args: - context (ScenarioContext): The resolved runtime inputs for this run. + template_file_name (str): The jailbreak template file name to deliver as a system prompt. Returns: - list[AtomicAttack]: List of atomic attacks to execute, one per jailbreak template. + AttackTechniqueFactory: A ``PromptSendingAttack`` factory whose system-role seed carries + the rendered jailbreak framing. """ - atomic_attacks: list[AtomicAttack] = [] + framing = TextJailBreak(template_file_name=template_file_name).get_jailbreak_system_prompt() + # sequence=-1 orders the system framing ahead of any user turn, so a caller-supplied seed + # group carrying a user prompt at the default sequence 0 does not raise a same-sequence + # role collision when this technique is merged in. + seed_technique = AttackTechniqueSeedGroup( + seeds=[SeedPrompt(value=framing, data_type="text", role="system", is_general_technique=True, sequence=-1)] + ) + return AttackTechniqueFactory( + name=_JAILBREAK_SYSTEM_PROMPT, + attack_class=PromptSendingAttack, + technique_tags=["single_turn"], + seed_technique=seed_technique, + ) - seed_groups = list(context.seed_groups) - techniques = {s.value for s in context.scenario_techniques} + @staticmethod + def _target_supports_system_delivery(target: PromptTarget) -> bool: + """ + Return whether ``target`` can carry the ``jailbreak_system_prompt`` delivery natively. - for technique in techniques: - for template_name in self._jailbreaks: - for _ in range(self._num_attempts): - atomic_attack = await self._get_atomic_attack_from_technique_async( - technique=technique, jailbreak_template_name=template_name, seed_groups=seed_groups - ) - atomic_attacks.append(atomic_attack) + System-prompt delivery sets a system prompt and relies on the objective staying a separate + live user turn; on a target without native editable history that turn is squashed into the + framing and the objective is silently dropped. Both capabilities must therefore be native. - return atomic_attacks + Args: + target (PromptTarget): The objective target for this run. + + Returns: + bool: ``True`` when the target natively supports editable history and system prompts. + """ + configuration = target.configuration + return configuration.includes(capability=CapabilityName.EDITABLE_HISTORY) and configuration.includes( + capability=CapabilityName.SYSTEM_PROMPT + ) diff --git a/pyrit/setup/initializers/techniques/extra.py b/pyrit/setup/initializers/techniques/extra.py index 9202d77840..8f01d5606d 100644 --- a/pyrit/setup/initializers/techniques/extra.py +++ b/pyrit/setup/initializers/techniques/extra.py @@ -10,7 +10,7 @@ """ from pyrit.common.path import EXECUTOR_RED_TEAM_PATH -from pyrit.executor.attack import PAIRAttack, RedTeamingAttack +from pyrit.executor.attack import PAIRAttack, RedTeamingAttack, SkeletonKeyAttack from pyrit.models import SeedPrompt from pyrit.scenario.core.attack_technique_factory import AttackTechniqueFactory @@ -28,6 +28,11 @@ def get_technique_factories() -> list[AttackTechniqueFactory]: attack_class=PAIRAttack, technique_tags=["multi_turn"], ), + AttackTechniqueFactory( + name="skeleton_key", + attack_class=SkeletonKeyAttack, + technique_tags=["single_turn"], + ), AttackTechniqueFactory( name="violent_durian", attack_class=RedTeamingAttack, diff --git a/tests/unit/scenario/airt/test_jailbreak.py b/tests/unit/scenario/airt/test_jailbreak.py index abea75293d..080d4a91c5 100644 --- a/tests/unit/scenario/airt/test_jailbreak.py +++ b/tests/unit/scenario/airt/test_jailbreak.py @@ -3,38 +3,74 @@ """Tests for the Jailbreak class.""" +from typing import Any from unittest.mock import AsyncMock, MagicMock, patch import pytest from pyrit.common.path import JAILBREAK_TEMPLATES_PATH +from pyrit.converter import TextJailbreakConverter from pyrit.datasets import TextJailBreak -from pyrit.executor.attack.single_turn.many_shot_jailbreak import ManyShotJailbreakAttack from pyrit.executor.attack.single_turn.prompt_sending import PromptSendingAttack -from pyrit.executor.attack.single_turn.skeleton_key import SkeletonKeyAttack -from pyrit.models import AttackSeedGroup, ComponentIdentifier, SeedObjective +from pyrit.models import AttackSeedGroup, ComponentIdentifier, SeedObjective, SeedPrompt from pyrit.prompt_target import PromptTarget +from pyrit.registry import TargetRegistry +from pyrit.registry.components.attack_technique_registry import AttackTechniqueRegistry from pyrit.scenario.core import BaselineAttackPolicy -from pyrit.scenario.scenarios.airt.jailbreak import Jailbreak, JailbreakTechnique +from pyrit.scenario.core.attack_technique_factory import AttackTechniqueFactory +from pyrit.scenario.scenarios.airt.jailbreak import ( + _DEFAULT_NUM_JAILBREAKS, + _DEFAULT_TECHNIQUES, + _JAILBREAK_SYSTEM_PROMPT, + _JAILBREAK_TEMPLATES_METADATA_KEY, + _PROMPT_SENDING, + Jailbreak, + _build_jailbreak_technique, +) from pyrit.score.true_false.true_false_inverter_scorer import TrueFalseInverterScorer +from pyrit.setup.initializers.techniques import build_technique_factories +# Synthetic many-shot examples — prevents reading the real JSON if a factory is constructed. +_MOCK_MANY_SHOT_EXAMPLES = [{"question": f"test question {i}", "answer": f"test answer {i}"} for i in range(100)] -@pytest.fixture -def mock_templates() -> list[str]: - """Mock constant for jailbreak subset.""" - return ["aim", "dan_1", "tuo"] +def _technique_class(): + """Get the dynamically-generated JailbreakTechnique class.""" + return _build_jailbreak_technique() -@pytest.fixture -def mock_random_num_attempts() -> int: - """Mock constant for n-many attempts per jailbreak.""" - return 2 +@pytest.fixture(autouse=True) +def reset_technique_registry(): + """Populate the attack-technique registry so the dynamic technique class can be built. + + Mirrors the RapidResponse test setup: reset the registries, register a mock adversarial + target (so factory construction does not fall back to a real target), and register the core + technique factories. The build cache is cleared around each test so the class reflects the + freshly-registered factories. + """ + AttackTechniqueRegistry.reset_registry_singleton() + TargetRegistry.reset_registry_singleton() + _build_jailbreak_technique.cache_clear() -@pytest.fixture -def mock_random_num_templates() -> int: - """Mock constant for k-many jailbreak templates to be used.""" - return 3 + adv_target = MagicMock(spec=PromptTarget) + adv_target.capabilities.includes.return_value = True + TargetRegistry.get_registry_singleton().instances.register(adv_target, name="adversarial_chat") + + AttackTechniqueRegistry.get_registry_singleton().register_from_factories(build_technique_factories()) + yield + AttackTechniqueRegistry.reset_registry_singleton() + TargetRegistry.reset_registry_singleton() + _build_jailbreak_technique.cache_clear() + + +@pytest.fixture(autouse=True) +def patch_many_shot_load(): + """Prevent ManyShotJailbreakAttack from loading the full bundled dataset.""" + with patch( + "pyrit.executor.attack.single_turn.many_shot_jailbreak.load_many_shot_jailbreaking_dataset", + return_value=_MOCK_MANY_SHOT_EXAMPLES, + ): + yield @pytest.fixture @@ -44,22 +80,32 @@ def mock_scenario_result_id() -> str: @pytest.fixture def mock_memory_seed_groups() -> list[AttackSeedGroup]: - """Create mock seed groups that _get_default_seed_groups() would return.""" + """Create mock seed groups that the dataset resolution would return.""" return [ AttackSeedGroup(seeds=[SeedObjective(value=prompt)]) - for prompt in [ - "sample objective 1", - "sample objective 2", - "sample objective 3", - ] + for prompt in ["sample objective 1", "sample objective 2", "sample objective 3"] ] @pytest.fixture def mock_objective_target() -> PromptTarget: - """Create a mock objective target for testing.""" + """Create a mock objective target that cannot carry native system-prompt delivery. + + ``configuration.includes(...)`` returns ``False`` so the default technique set degrades to the + target-agnostic ``prompt_sending`` delivery (the ``jailbreak_system_prompt`` delivery is skipped). + """ mock = MagicMock(spec=PromptTarget) mock.get_identifier.return_value = ComponentIdentifier(class_name="MockObjectiveTarget", class_module="test") + mock.configuration.includes.return_value = False + return mock + + +@pytest.fixture +def mock_capable_target() -> PromptTarget: + """Create a mock objective target that natively supports editable history + system prompts.""" + mock = MagicMock(spec=PromptTarget) + mock.get_identifier.return_value = ComponentIdentifier(class_name="MockCapableTarget", class_module="test") + mock.configuration.includes.return_value = True return mock @@ -71,55 +117,6 @@ def mock_objective_scorer() -> TrueFalseInverterScorer: return mock -@pytest.fixture -def all_jailbreak_technique() -> JailbreakTechnique: - return JailbreakTechnique.ALL - - -@pytest.fixture -def simple_jailbreak_technique() -> JailbreakTechnique: - return JailbreakTechnique.SIMPLE - - -@pytest.fixture -def complex_jailbreak_technique() -> JailbreakTechnique: - return JailbreakTechnique.COMPLEX - - -@pytest.fixture -def manyshot_jailbreak_technique() -> JailbreakTechnique: - return JailbreakTechnique.ManyShot - - -@pytest.fixture -def promptsending_jailbreak_technique() -> JailbreakTechnique: - return JailbreakTechnique.PromptSending - - -@pytest.fixture -def skeleton_jailbreak_attack() -> JailbreakTechnique: - return JailbreakTechnique.SkeletonKey - - -@pytest.fixture -def roleplay_jailbreak_technique() -> JailbreakTechnique: - return JailbreakTechnique.RolePlay - - -# Synthetic many-shot examples used to prevent real HTTP requests to GitHub during tests -_MOCK_MANY_SHOT_EXAMPLES = [{"question": f"test question {i}", "answer": f"test answer {i}"} for i in range(100)] - - -@pytest.fixture(autouse=True) -def patch_many_shot_load(): - """Prevent ManyShotJailbreakAttack from loading the full dataset during unit tests.""" - with patch( - "pyrit.executor.attack.single_turn.many_shot_jailbreak.load_many_shot_jailbreaking_dataset", - return_value=_MOCK_MANY_SHOT_EXAMPLES, - ): - yield - - @pytest.fixture def mock_runtime_env(): with patch.dict( @@ -139,98 +136,113 @@ def mock_runtime_env(): FIXTURES = ["patch_central_database", "mock_runtime_env"] +def _patch_seed_groups(seed_groups): + return patch.object( + Jailbreak, + "_resolve_seed_groups_by_dataset_async", + new_callable=AsyncMock, + return_value={"harmbench": seed_groups}, + ) + + +def _default_args(target, **extra): + """Run args selecting the default technique (``prompt_sending``, "just send").""" + technique_class = _build_jailbreak_technique() + args = { + "objective_target": target, + "scenario_techniques": [technique_class("default")], + "include_baseline": False, + } + args.update(extra) + return args + + @pytest.mark.usefixtures(*FIXTURES) class TestJailbreakInitialization: """Tests for Jailbreak initialization.""" - def test_init_with_scenario_result_id(self, mock_scenario_result_id): - """Test initialization with a scenario result ID.""" - with patch.object( - Jailbreak, - "_resolve_seed_groups_by_dataset_async", - new_callable=AsyncMock, - return_value={"memory": mock_memory_seed_groups}, - ): + def test_init_with_scenario_result_id(self, mock_scenario_result_id, mock_memory_seed_groups): + with _patch_seed_groups(mock_memory_seed_groups): scenario = Jailbreak(scenario_result_id=mock_scenario_result_id) assert scenario._scenario_result_id == mock_scenario_result_id def test_init_with_default_scorer(self, mock_memory_seed_groups): - """Test initialization with default scorer.""" - with patch.object( - Jailbreak, - "_resolve_seed_groups_by_dataset_async", - new_callable=AsyncMock, - return_value={"memory": mock_memory_seed_groups}, - ): + with _patch_seed_groups(mock_memory_seed_groups): scenario = Jailbreak() assert scenario._objective_scorer_identifier def test_init_with_custom_scorer(self, mock_objective_scorer, mock_memory_seed_groups): - """Test initialization with custom scorer.""" - with patch.object( - Jailbreak, - "_resolve_seed_groups_by_dataset_async", - new_callable=AsyncMock, - return_value={"memory": mock_memory_seed_groups}, - ): + with _patch_seed_groups(mock_memory_seed_groups): scenario = Jailbreak(objective_scorer=mock_objective_scorer) assert scenario._objective_scorer == mock_objective_scorer - def test_init_with_num_templates(self, mock_random_num_templates): - """Test initialization with num_templates provided.""" - with patch.object( - Jailbreak, - "_resolve_seed_groups_by_dataset_async", - new_callable=AsyncMock, - return_value={"memory": mock_memory_seed_groups}, - ): - scenario = Jailbreak(num_templates=mock_random_num_templates) - assert scenario._num_templates == mock_random_num_templates - - def test_init_with_num_attempts(self, mock_random_num_attempts): - """Test initialization with n provided.""" - with patch.object( - Jailbreak, - "_resolve_seed_groups_by_dataset_async", - new_callable=AsyncMock, - return_value={"memory": mock_memory_seed_groups}, - ): - scenario = Jailbreak(num_attempts=mock_random_num_attempts) - assert scenario._num_attempts == mock_random_num_attempts + def test_declares_run_parameters(self): + """num_jailbreaks / num_jailbreak_attempts / jailbreak_names are declared as run parameters.""" + names = {p.name for p in Jailbreak.additional_parameters()} + assert names == {"num_jailbreaks", "num_jailbreak_attempts", "jailbreak_names"} + assert set(names).issubset({p.name for p in Jailbreak.supported_parameters()}) - def test_init_raises_exception_when_both_num_and_which_jailbreaks(self, mock_random_num_templates, mock_templates): - """Test failure on providing mutually exclusive arguments.""" + async def test_default_draws_random_template_sample( + self, mock_objective_target, mock_objective_scorer, mock_memory_seed_groups + ): + """A bare run draws a small random sample of templates (no curated default set).""" + with _patch_seed_groups(mock_memory_seed_groups): + scenario = Jailbreak(objective_scorer=mock_objective_scorer) + scenario.set_params_from_args(args=_default_args(mock_objective_target)) + await scenario.initialize_async() + assert len(scenario._resolved_jailbreaks) == _DEFAULT_NUM_JAILBREAKS + assert set(scenario._resolved_jailbreaks).issubset(set(TextJailBreak.get_jailbreak_templates())) - with pytest.raises(ValueError): - Jailbreak(num_templates=mock_random_num_templates, jailbreak_names=mock_templates) + async def test_num_jailbreaks_samples_that_many( + self, mock_objective_target, mock_objective_scorer, mock_memory_seed_groups + ): + with _patch_seed_groups(mock_memory_seed_groups): + scenario = Jailbreak(objective_scorer=mock_objective_scorer) + scenario.set_params_from_args(args=_default_args(mock_objective_target, num_jailbreaks=3)) + await scenario.initialize_async() + assert len(scenario._resolved_jailbreaks) == 3 + + async def test_mutually_exclusive_selectors_raise( + self, mock_objective_target, mock_objective_scorer, mock_memory_seed_groups + ): + with _patch_seed_groups(mock_memory_seed_groups): + scenario = Jailbreak(objective_scorer=mock_objective_scorer) + scenario.set_params_from_args( + args=_default_args(mock_objective_target, num_jailbreaks=2, jailbreak_names=["aim.yaml"]) + ) + with pytest.raises(ValueError, match="only one of"): + await scenario.initialize_async() + + async def test_unknown_jailbreak_name_raises( + self, mock_objective_target, mock_objective_scorer, mock_memory_seed_groups + ): + with _patch_seed_groups(mock_memory_seed_groups): + scenario = Jailbreak(objective_scorer=mock_objective_scorer) + scenario.set_params_from_args( + args=_default_args(mock_objective_target, jailbreak_names=["definitely_not_real.yaml"]) + ) + with pytest.raises(ValueError, match="could not find templates"): + await scenario.initialize_async() - def test_init_accepts_subdirectory_jailbreak_names(self, mock_objective_scorer, mock_memory_seed_groups): - """Test that explicit jailbreak names can reference templates stored in subdirectories.""" - # Pick a template that lives in a subdirectory (not top-level) + async def test_accepts_subdirectory_jailbreak_names( + self, mock_objective_target, mock_objective_scorer, mock_memory_seed_groups + ): all_templates = TextJailBreak.get_jailbreak_templates() top_level_names = {f.name for f in JAILBREAK_TEMPLATES_PATH.glob("*.yaml")} subdir_templates = [t for t in all_templates if t not in top_level_names] assert subdir_templates, "Expected at least one subdirectory template to exist" subdir_name = subdir_templates[0] - with patch.object( - Jailbreak, - "_resolve_seed_groups_by_dataset_async", - new_callable=AsyncMock, - return_value={"memory": mock_memory_seed_groups}, - ): - scenario = Jailbreak(objective_scorer=mock_objective_scorer, jailbreak_names=[subdir_name]) - assert scenario._jailbreaks == [subdir_name] + with _patch_seed_groups(mock_memory_seed_groups): + scenario = Jailbreak(objective_scorer=mock_objective_scorer) + scenario.set_params_from_args(args=_default_args(mock_objective_target, jailbreak_names=[subdir_name])) + await scenario.initialize_async() + assert scenario._resolved_jailbreaks == [subdir_name] async def test_init_raises_exception_when_no_datasets_available(self, mock_objective_target, mock_objective_scorer): - """Test that initialization raises DatasetConstraintError when datasets are not available in memory.""" from pyrit.scenario.core.dataset_configuration import DatasetConstraintError - # Don't mock _resolve_seed_groups, let it try to load from empty memory scenario = Jailbreak(objective_scorer=mock_objective_scorer) - - # Error should occur during initialize_async when _get_atomic_attacks_async resolves seed groups. - # Neutralize the provider fetch so the empty-memory path raises loudly instead of fetching. with patch( "pyrit.scenario.core.dataset_configuration.DatasetConfiguration._fetch_dataset_async", new_callable=AsyncMock, @@ -239,477 +251,471 @@ async def test_init_raises_exception_when_no_datasets_available(self, mock_objec with pytest.raises(DatasetConstraintError, match="could not be loaded"): await scenario.initialize_async() - def test_class_inherits_default_baseline_attack_policy(self): - """Jailbreak inherits the base default (Enabled) — baseline included by default.""" + def test_class_uses_enabled_baseline_attack_policy(self): assert Jailbreak.BASELINE_ATTACK_POLICY is BaselineAttackPolicy.Enabled async def test_default_initialize_includes_baseline( self, mock_objective_target, mock_objective_scorer, mock_memory_seed_groups ): - """initialize_async without include_baseline honors BASELINE_ATTACK_POLICY=Enabled.""" - with patch.object( - Jailbreak, - "_resolve_seed_groups_by_dataset_async", - new_callable=AsyncMock, - return_value={"memory": mock_memory_seed_groups}, - ): + """Baseline is on by default (BASELINE_ATTACK_POLICY = Enabled): a bare run prepends one baseline.""" + with _patch_seed_groups(mock_memory_seed_groups): scenario = Jailbreak(objective_scorer=mock_objective_scorer) scenario.set_params_from_args(args={"objective_target": mock_objective_target}) await scenario.initialize_async() assert scenario._atomic_attacks[0].atomic_attack_name == "baseline" + assert sum(a.atomic_attack_name == "baseline" for a in scenario._atomic_attacks) == 1 + + async def test_include_baseline_false_opts_out( + self, mock_objective_target, mock_objective_scorer, mock_memory_seed_groups + ): + """An Enabled policy still honours an explicit ``include_baseline=False`` opt-out.""" + with _patch_seed_groups(mock_memory_seed_groups): + scenario = Jailbreak(objective_scorer=mock_objective_scorer) + scenario.set_params_from_args(args={"objective_target": mock_objective_target, "include_baseline": False}) + await scenario.initialize_async() + assert not any(a.atomic_attack_name == "baseline" for a in scenario._atomic_attacks) async def test_explicit_include_baseline_false_omits_baseline( self, mock_objective_target, mock_objective_scorer, mock_memory_seed_groups ): - """Caller can opt out of baseline by passing include_baseline=False.""" - with patch.object( - Jailbreak, - "_resolve_seed_groups_by_dataset_async", - new_callable=AsyncMock, - return_value={"memory": mock_memory_seed_groups}, - ): + with _patch_seed_groups(mock_memory_seed_groups): scenario = Jailbreak(objective_scorer=mock_objective_scorer) - scenario.set_params_from_args( - args={ - "objective_target": mock_objective_target, - "include_baseline": False, - } - ) + scenario.set_params_from_args(args=_default_args(mock_objective_target)) await scenario.initialize_async() assert not any(a.atomic_attack_name == "baseline" for a in scenario._atomic_attacks) @pytest.mark.usefixtures(*FIXTURES) class TestJailbreakAttackGeneration: - """Tests for Jailbreak attack generation.""" + """Tests for Jailbreak atomic-attack generation.""" - async def test_attack_generation_for_simple( - self, mock_objective_target, mock_objective_scorer, mock_memory_seed_groups, simple_jailbreak_technique + async def test_default_builds_prompt_sending_per_template( + self, mock_objective_target, mock_objective_scorer, mock_memory_seed_groups ): - """Test that the simple attack generation works.""" - with patch.object( - Jailbreak, - "_resolve_seed_groups_by_dataset_async", - new_callable=AsyncMock, - return_value={"memory": mock_memory_seed_groups}, - ): - scenario = Jailbreak(objective_scorer=mock_objective_scorer, num_templates=2) - - scenario.set_params_from_args( - args={ - "objective_target": mock_objective_target, - "scenario_techniques": [simple_jailbreak_technique], - } - ) + """On a target without native system-prompt support, the default degrades to a single + ``prompt_sending`` ("just send") delivery, one atomic attack per template.""" + with _patch_seed_groups(mock_memory_seed_groups): + scenario = Jailbreak(objective_scorer=mock_objective_scorer) + scenario.set_params_from_args(args=_default_args(mock_objective_target, jailbreak_names=["aim.yaml"])) await scenario.initialize_async() - atomic_attacks = scenario._atomic_attacks - for run in atomic_attacks: - assert isinstance(run.attack_technique.attack, PromptSendingAttack) + attacks = scenario._atomic_attacks + assert len(attacks) == 1 + assert isinstance(attacks[0].attack_technique.attack, PromptSendingAttack) + assert attacks[0].atomic_attack_name == f"{_PROMPT_SENDING}_aim_harmbench" - async def test_attack_generation_for_complex( - self, mock_objective_target, mock_objective_scorer, mock_memory_seed_groups, complex_jailbreak_technique + async def test_jailbreak_delivered_as_request_converter( + self, mock_objective_target, mock_objective_scorer, mock_memory_seed_groups ): - """Test that the complex attack generation works.""" - with patch.object( - Jailbreak, - "_resolve_seed_groups_by_dataset_async", - new_callable=AsyncMock, - return_value={"memory": mock_memory_seed_groups}, - ): - scenario = Jailbreak(objective_scorer=mock_objective_scorer, num_templates=2) + """The crux: the jailbreak template reaches the target as a ``TextJailbreakConverter`` on + the technique's outgoing requests (not as prepended framing on the seed group). - scenario.set_params_from_args( - args={ - "objective_target": mock_objective_target, - "scenario_techniques": [complex_jailbreak_technique], - "include_baseline": False, - } + Delivery via ``factory.create(extra_request_converters=...)`` is what keeps the scenario + target-agnostic and composable with every technique. Also assert the seed groups carry no + prepended jailbreak framing. + """ + captured: list[Any] = [] + original_create = AttackTechniqueFactory.create + + def _spy_create(self, **kwargs): + captured.append(kwargs.get("extra_request_converters")) + return original_create(self, **kwargs) + + with _patch_seed_groups(mock_memory_seed_groups): + with patch.object(AttackTechniqueFactory, "create", _spy_create): + scenario = Jailbreak(objective_scorer=mock_objective_scorer) + scenario.set_params_from_args(args=_default_args(mock_objective_target, jailbreak_names=["aim.yaml"])) + await scenario.initialize_async() + + assert captured, "Expected factory.create to be called" + converters = [c for extra in captured if extra for cc in extra for c in cc.converters] + assert any(isinstance(c, TextJailbreakConverter) for c in converters), ( + "Expected a TextJailbreakConverter to be threaded to factory.create" ) - await scenario.initialize_async() - atomic_attacks = scenario._atomic_attacks - for run in atomic_attacks: - assert isinstance( - run.attack_technique.attack, (PromptSendingAttack, ManyShotJailbreakAttack, SkeletonKeyAttack) - ) - async def test_attack_generation_for_manyshot( - self, mock_objective_target, mock_objective_scorer, mock_memory_seed_groups, manyshot_jailbreak_technique + # The objective seed groups themselves carry no jailbreak framing (converter delivery only). + for attack in scenario._atomic_attacks: + for group in attack.seed_groups: + assert not group.prepended_conversation + + async def test_user_technique_converters_are_preserved_alongside_jailbreak( + self, mock_objective_target, mock_objective_scorer, mock_memory_seed_groups ): - """Test that the manyshot attack generation works.""" - with patch.object( - Jailbreak, - "_resolve_seed_groups_by_dataset_async", - new_callable=AsyncMock, - return_value={"memory": mock_memory_seed_groups}, - ): - scenario = Jailbreak(objective_scorer=mock_objective_scorer, num_templates=2) + """A caller-supplied ``--techniques prompt_sending:converter`` stack is not dropped: the + jailbreak converter is layered on top of (not in place of) the user's converters.""" + from pyrit.converter import Base64Converter + + technique_class = _build_jailbreak_technique() + user_converter = Base64Converter() + captured: list[Any] = [] + original_create = AttackTechniqueFactory.create + + def _spy_create(self, **kwargs): + captured.append(kwargs.get("extra_request_converters")) + return original_create(self, **kwargs) + + with _patch_seed_groups(mock_memory_seed_groups): + with patch.object(AttackTechniqueFactory, "create", _spy_create): + scenario = Jailbreak(objective_scorer=mock_objective_scorer) + scenario.set_params_from_args( + args=_default_args( + mock_objective_target, + jailbreak_names=["aim.yaml"], + technique_converters={_PROMPT_SENDING: [user_converter]}, + ) + ) + await scenario.initialize_async() - scenario.set_params_from_args( - args={ - "objective_target": mock_objective_target, - "scenario_techniques": [manyshot_jailbreak_technique], - "include_baseline": False, - } + converters = [c for extra in captured if extra for cc in extra for c in cc.converters] + jailbreak_indices = [i for i, c in enumerate(converters) if isinstance(c, TextJailbreakConverter)] + user_indices = [i for i, c in enumerate(converters) if c is user_converter] + assert jailbreak_indices, "Expected a TextJailbreakConverter in the stack" + assert user_indices, "User-supplied converter must be preserved" + # The jailbreak must wrap the raw objective before any caller converters transform it, so + # the jailbreak converter has to precede the user converter in the extra-converter stack. + assert max(jailbreak_indices) < min(user_indices), ( + "Jailbreak converter must be applied before caller-supplied converters" ) - await scenario.initialize_async() - atomic_attacks = scenario._atomic_attacks - for run in atomic_attacks: - assert isinstance(run.attack_technique.attack, ManyShotJailbreakAttack) - async def test_attack_generation_for_promptsending( - self, mock_objective_target, mock_objective_scorer, mock_memory_seed_groups, promptsending_jailbreak_technique + async def test_simulated_conversation_techniques_produce_attacks_with_jailbreak( + self, mock_objective_target, mock_objective_scorer, mock_memory_seed_groups ): - """Test that the prompt sending attack generation works.""" - with patch.object( - Jailbreak, - "_resolve_seed_groups_by_dataset_async", - new_callable=AsyncMock, - return_value={"memory": mock_memory_seed_groups}, - ): - scenario = Jailbreak(objective_scorer=mock_objective_scorer, num_templates=2) + """Regression: simulated-conversation techniques (``role_play_*``, ``crescendo_*``) must still + produce atomic attacks when crossed with a jailbreak template, and each must receive the + jailbreak converter. + Converter delivery leaves the objective seed group unframed, so it stays compatible with the + simulated-conversation seed technique. (Delivering the jailbreak as a system-role framing seed + instead collided with that technique's seed range and silently produced zero attacks.) + """ + technique_class = _build_jailbreak_technique() + techniques = [ + technique_class("role_play_movie_script"), + technique_class("crescendo_simulated"), + ] + captured: list[Any] = [] + original_create = AttackTechniqueFactory.create + + def _spy_create(self, **kwargs): + captured.append(kwargs.get("extra_request_converters")) + return original_create(self, **kwargs) + + with _patch_seed_groups(mock_memory_seed_groups): + with patch.object(AttackTechniqueFactory, "create", _spy_create): + scenario = Jailbreak(objective_scorer=mock_objective_scorer) + scenario.set_params_from_args( + args=_default_args( + mock_objective_target, scenario_techniques=techniques, jailbreak_names=["aim.yaml"] + ) + ) + await scenario.initialize_async() + names = {a.atomic_attack_name for a in scenario._atomic_attacks} + assert "role_play_movie_script_aim_harmbench" in names + assert "crescendo_simulated_aim_harmbench" in names + # Every build of a simulated-conversation technique must still carry the jailbreak + # converter. Assert both techniques captured a non-empty converter stack (so the check + # can't pass vacuously on a dropped/None stack) and each contains the jailbreak converter. + populated = [extra for extra in captured if extra] + assert len(populated) == 2, "Expected both simulated-conversation techniques to receive converters" + assert all( + any(isinstance(c, TextJailbreakConverter) for cc in extra for c in cc.converters) for extra in populated + ), "Each simulated-conversation technique must receive the jailbreak converter" + + async def test_all_templates_produce_attacks( + self, mock_objective_target, mock_objective_scorer, mock_memory_seed_groups + ): + """Each selected template yields its own atomic attack, grouped by template stem.""" + with _patch_seed_groups(mock_memory_seed_groups): + scenario = Jailbreak(objective_scorer=mock_objective_scorer) scenario.set_params_from_args( - args={ - "objective_target": mock_objective_target, - "scenario_techniques": [promptsending_jailbreak_technique], - "include_baseline": False, - } + args=_default_args(mock_objective_target, jailbreak_names=["aim.yaml", "dan_11.yaml"]) ) await scenario.initialize_async() - atomic_attacks = scenario._atomic_attacks - for run in atomic_attacks: - assert isinstance(run.attack_technique.attack, PromptSendingAttack) + names = {a.atomic_attack_name for a in scenario._atomic_attacks} + assert names == { + f"{_PROMPT_SENDING}_aim_harmbench", + f"{_PROMPT_SENDING}_dan_11_harmbench", + } - async def test_attack_generation_for_skeleton( - self, mock_objective_target, mock_objective_scorer, mock_memory_seed_groups, skeleton_jailbreak_attack + async def test_display_group_is_template_stem( + self, mock_objective_target, mock_objective_scorer, mock_memory_seed_groups ): - """Test that the skelton key attack generation works.""" - with patch.object( - Jailbreak, - "_resolve_seed_groups_by_dataset_async", - new_callable=AsyncMock, - return_value={"memory": mock_memory_seed_groups}, - ): - scenario = Jailbreak(objective_scorer=mock_objective_scorer, num_templates=2) - + with _patch_seed_groups(mock_memory_seed_groups): + scenario = Jailbreak(objective_scorer=mock_objective_scorer) scenario.set_params_from_args( - args={ - "objective_target": mock_objective_target, - "scenario_techniques": [skeleton_jailbreak_attack], - "include_baseline": False, - } + args=_default_args(mock_objective_target, jailbreak_names=["aim.yaml", "dan_11.yaml"]) ) await scenario.initialize_async() - atomic_attacks = scenario._atomic_attacks - for run in atomic_attacks: - assert isinstance(run.attack_technique.attack, SkeletonKeyAttack) + groups = {a.display_group for a in scenario._atomic_attacks} + assert groups == {"aim", "dan_11"} - async def test_attack_generation_for_roleplay( - self, mock_objective_target, mock_objective_scorer, mock_memory_seed_groups, roleplay_jailbreak_technique + async def test_atomic_attack_names_are_unique( + self, mock_objective_target, mock_objective_scorer, mock_memory_seed_groups ): - """Test that the roleplaying attack generation works.""" - with patch.object( - Jailbreak, - "_resolve_seed_groups_by_dataset_async", - new_callable=AsyncMock, - return_value={"memory": mock_memory_seed_groups}, - ): - scenario = Jailbreak(objective_scorer=mock_objective_scorer, num_templates=2) - + with _patch_seed_groups(mock_memory_seed_groups): + scenario = Jailbreak(objective_scorer=mock_objective_scorer) scenario.set_params_from_args( - args={ - "objective_target": mock_objective_target, - "scenario_techniques": [roleplay_jailbreak_technique], - "include_baseline": False, - } + args=_default_args( + mock_objective_target, jailbreak_names=["aim.yaml", "dan_11.yaml"], num_jailbreak_attempts=2 + ) ) await scenario.initialize_async() - atomic_attacks = scenario._atomic_attacks - for run in atomic_attacks: - assert isinstance(run.attack_technique.attack, PromptSendingAttack) - assert run.attack_technique.seed_technique is not None - assert run.attack_technique.seed_technique.has_simulated_conversation + names = [a.atomic_attack_name for a in scenario._atomic_attacks] + assert len(names) == len(set(names)) + assert len(names) == 4 # 2 templates x 2 attempts - async def test_attack_runs_include_objectives( + async def test_memory_labels_propagate_to_atomic_attacks( self, mock_objective_target, mock_objective_scorer, mock_memory_seed_groups ): - """Test that attack runs include objectives for each seed prompt and that - each atomic attack carries a valid attack_technique. - - Combined coverage previously split across test_get_atomic_attacks_async_returns_attacks. - """ - with patch.object( - Jailbreak, - "_resolve_seed_groups_by_dataset_async", - new_callable=AsyncMock, - return_value={"memory": mock_memory_seed_groups}, - ): - scenario = Jailbreak(objective_scorer=mock_objective_scorer, num_templates=2) - - scenario.set_params_from_args(args={"objective_target": mock_objective_target}) + """Run-level memory_labels reach every built atomic attack (matches the sibling convention).""" + labels = {"experiment": "jb-run", "operator": "airt"} + with _patch_seed_groups(mock_memory_seed_groups): + scenario = Jailbreak(objective_scorer=mock_objective_scorer) + scenario.set_params_from_args( + args=_default_args( + mock_objective_target, jailbreak_names=["aim.yaml", "dan_11.yaml"], memory_labels=labels + ) + ) await scenario.initialize_async() - atomic_attacks = scenario._atomic_attacks - - assert len(atomic_attacks) > 0 - for run in atomic_attacks: - assert run.attack_technique is not None - assert len(run.objectives) > 0 + assert scenario._atomic_attacks + assert all(a._memory_labels == labels for a in scenario._atomic_attacks) - async def test_get_all_jailbreak_templates( + async def test_num_attempts_multiplies_atomic_attacks( self, mock_objective_target, mock_objective_scorer, mock_memory_seed_groups ): - """Test that all jailbreak templates are found.""" - with patch.object( - Jailbreak, - "_resolve_seed_groups_by_dataset_async", - new_callable=AsyncMock, - return_value={"memory": mock_memory_seed_groups}, - ): - scenario = Jailbreak( - objective_scorer=mock_objective_scorer, + with _patch_seed_groups(mock_memory_seed_groups): + scenario = Jailbreak(objective_scorer=mock_objective_scorer) + scenario.set_params_from_args( + args=_default_args(mock_objective_target, jailbreak_names=["aim.yaml"], num_jailbreak_attempts=3) ) - scenario.set_params_from_args(args={"objective_target": mock_objective_target}) await scenario.initialize_async() - assert len(scenario._jailbreaks) > 0 + assert len(scenario._atomic_attacks) == 3 + - async def test_get_some_jailbreak_templates( - self, mock_objective_target, mock_objective_scorer, mock_memory_seed_groups, mock_random_num_templates +@pytest.mark.usefixtures(*FIXTURES) +class TestJailbreakSystemPromptDelivery: + """Tests for the ``jailbreak_system_prompt`` native system-prompt delivery (J5).""" + + async def test_capable_target_builds_both_deliveries( + self, mock_capable_target, mock_objective_scorer, mock_memory_seed_groups ): - """Test that random jailbreak template selection works.""" - with patch.object( - Jailbreak, - "_resolve_seed_groups_by_dataset_async", - new_callable=AsyncMock, - return_value={"memory": mock_memory_seed_groups}, - ): - scenario = Jailbreak(objective_scorer=mock_objective_scorer, num_templates=mock_random_num_templates) - scenario.set_params_from_args(args={"objective_target": mock_objective_target}) + """On a capable target the default set builds both deliveries for each template.""" + with _patch_seed_groups(mock_memory_seed_groups): + scenario = Jailbreak(objective_scorer=mock_objective_scorer) + scenario.set_params_from_args(args=_default_args(mock_capable_target, jailbreak_names=["aim.yaml"])) await scenario.initialize_async() - assert len(scenario._jailbreaks) == mock_random_num_templates - - async def test_custom_num_attempts( - self, mock_objective_target, mock_objective_scorer, mock_memory_seed_groups, mock_random_num_attempts + names = {a.atomic_attack_name for a in scenario._atomic_attacks} + assert names == { + f"{_PROMPT_SENDING}_aim_harmbench", + f"{_JAILBREAK_SYSTEM_PROMPT}_aim_harmbench", + } + + async def test_system_delivery_attaches_system_role_framing_seed( + self, mock_capable_target, mock_objective_scorer, mock_memory_seed_groups ): - """Test that n successfully tries each jailbreak template n-many times.""" - with patch.object( - Jailbreak, - "_resolve_seed_groups_by_dataset_async", - new_callable=AsyncMock, - return_value={"memory": mock_memory_seed_groups}, - ): - base_scenario = Jailbreak(objective_scorer=mock_objective_scorer, num_templates=2) - base_scenario.set_params_from_args( - args={ - "objective_target": mock_objective_target, - "include_baseline": False, - } + """The system-prompt delivery carries the jailbreak framing as a ``role="system"`` seed.""" + with _patch_seed_groups(mock_memory_seed_groups): + scenario = Jailbreak(objective_scorer=mock_objective_scorer) + scenario.set_params_from_args(args=_default_args(mock_capable_target, jailbreak_names=["aim.yaml"])) + await scenario.initialize_async() + system_attack = next( + a + for a in scenario._atomic_attacks + if a.atomic_attack_name == f"{_JAILBREAK_SYSTEM_PROMPT}_aim_harmbench" ) - await base_scenario.initialize_async() - atomic_attacks_1 = base_scenario._atomic_attacks + seed_technique = system_attack.attack_technique.seed_technique + assert seed_technique is not None + assert [s.role for s in seed_technique.seeds] == ["system"] + assert seed_technique.seeds[0].value - mult_scenario = Jailbreak( - objective_scorer=mock_objective_scorer, - num_templates=2, - num_attempts=mock_random_num_attempts, - ) - mult_scenario.set_params_from_args( - args={ - "objective_target": mock_objective_target, - "include_baseline": False, - } - ) - await mult_scenario.initialize_async() - atomic_attacks_n = mult_scenario._atomic_attacks + async def test_system_delivery_uses_no_jailbreak_converter( + self, mock_capable_target, mock_objective_scorer, mock_memory_seed_groups + ): + """The jailbreak is delivered once: the converter path carries the ``TextJailbreakConverter``, + the system-prompt path carries none (so the framing is not double-applied).""" + captured: list[tuple[str, Any]] = [] + original_create = AttackTechniqueFactory.create + + def _spy_create(self, **kwargs): + captured.append((self.name, kwargs.get("extra_request_converters"))) + return original_create(self, **kwargs) + + with _patch_seed_groups(mock_memory_seed_groups): + with patch.object(AttackTechniqueFactory, "create", _spy_create): + scenario = Jailbreak(objective_scorer=mock_objective_scorer) + scenario.set_params_from_args(args=_default_args(mock_capable_target, jailbreak_names=["aim.yaml"])) + await scenario.initialize_async() - assert len(atomic_attacks_1) * mock_random_num_attempts == len(atomic_attacks_n) + by_name = dict(captured) + assert _PROMPT_SENDING in by_name and _JAILBREAK_SYSTEM_PROMPT in by_name + # System delivery gets no extra converter. + assert by_name[_JAILBREAK_SYSTEM_PROMPT] is None + # Converter delivery gets the jailbreak converter. + prompt_sending_converters = [c for cc in by_name[_PROMPT_SENDING] for c in cc.converters] + assert any(isinstance(c, TextJailbreakConverter) for c in prompt_sending_converters) + async def test_incapable_target_degrades_to_converter_delivery( + self, mock_objective_target, mock_objective_scorer, mock_memory_seed_groups, caplog + ): + """When the target can't carry native system delivery, the default degrades to + ``prompt_sending`` only and logs a warning (rather than failing).""" + import logging -@pytest.mark.usefixtures(*FIXTURES) -class TestJailbreakLifecycle: - """Tests for Jailbreak lifecycle.""" - - async def test_initialize_async_with_max_concurrency( - self, - *, - mock_objective_target: PromptTarget, - mock_objective_scorer: TrueFalseInverterScorer, - mock_memory_seed_groups: list[AttackSeedGroup], - ) -> None: - """Test initialization with custom max_concurrency.""" - with patch.object( - Jailbreak, - "_resolve_seed_groups_by_dataset_async", - new_callable=AsyncMock, - return_value={"memory": mock_memory_seed_groups}, - ): + with _patch_seed_groups(mock_memory_seed_groups): scenario = Jailbreak(objective_scorer=mock_objective_scorer) - scenario.set_params_from_args( - args={ - "objective_target": mock_objective_target, - "max_concurrency": 20, - } - ) - await scenario.initialize_async() - assert scenario._max_concurrency == 20 - - async def test_initialize_async_with_memory_labels( - self, - *, - mock_objective_target: PromptTarget, - mock_objective_scorer: TrueFalseInverterScorer, - mock_memory_seed_groups: list[AttackSeedGroup], - ) -> None: - """Test initialization with memory labels.""" - memory_labels = {"type": "jailbreak", "category": "scenario"} - - with patch.object( - Jailbreak, - "_resolve_seed_groups_by_dataset_async", - new_callable=AsyncMock, - return_value={"memory": mock_memory_seed_groups}, - ): + scenario.set_params_from_args(args=_default_args(mock_objective_target, jailbreak_names=["aim.yaml"])) + with caplog.at_level(logging.WARNING): + await scenario.initialize_async() + names = {a.atomic_attack_name for a in scenario._atomic_attacks} + assert names == {f"{_PROMPT_SENDING}_aim_harmbench"} + assert "jailbreak_system_prompt" in caplog.text + + async def test_system_only_incapable_target_raises( + self, mock_objective_target, mock_objective_scorer, mock_memory_seed_groups + ): + """If ``jailbreak_system_prompt`` is the only selected technique and the target can't carry + it, initialization raises a clear error (no silent no-op run).""" + technique_class = _build_jailbreak_technique() + with _patch_seed_groups(mock_memory_seed_groups): scenario = Jailbreak(objective_scorer=mock_objective_scorer) scenario.set_params_from_args( - args={ - "memory_labels": memory_labels, - "objective_target": mock_objective_target, - } + args=_default_args( + mock_objective_target, + scenario_techniques=[technique_class(_JAILBREAK_SYSTEM_PROMPT)], + jailbreak_names=["aim.yaml"], + ) ) - await scenario.initialize_async() - assert scenario._memory_labels == memory_labels - + with pytest.raises(ValueError, match="natively supports"): + await scenario.initialize_async() -@pytest.mark.usefixtures(*FIXTURES) -class TestJailbreakProperties: - """Tests for Jailbreak properties.""" - - def test_scenario_version_is_set( - self, - *, - mock_objective_scorer: TrueFalseInverterScorer, - ) -> None: - """Test that scenario version is properly set.""" - scenario = Jailbreak( - objective_scorer=mock_objective_scorer, - ) - - assert scenario.VERSION == 1 - - def test_scenario_default_dataset(self) -> None: - """Test that scenario default dataset is correct.""" - - assert Jailbreak.required_datasets() == ["airt_harms"] - - async def test_no_target_duplication_async( - self, *, mock_objective_target: PromptTarget, mock_memory_seed_groups: list[AttackSeedGroup] - ) -> None: - """Test that all three targets (adversarial, object, scorer) are distinct.""" - with patch.object( - Jailbreak, - "_resolve_seed_groups_by_dataset_async", - new_callable=AsyncMock, - return_value={"memory": mock_memory_seed_groups}, - ): - scenario = Jailbreak() - scenario.set_params_from_args(args={"objective_target": mock_objective_target}) - await scenario.initialize_async() + async def test_system_delivery_end_to_end_keeps_objective_live(self, mock_memory_seed_groups): + """End-to-end: on a capable target the framing is delivered as a system message and the + objective as a clean live user turn (never dropped, never wrapped by the jailbreak template). - objective_target = scenario._objective_target - scorer_target = scenario._objective_scorer + This guards the novel capability-gated delivery against the objective-drop failure mode: a + system-role framing seed leaves the seed group with no ``next_message``, so ``PromptSendingAttack`` + must fall back to sending the objective itself as the user turn. + """ + from pyrit.memory import CentralMemory + from pyrit.score import SubStringScorer + from tests.unit.mocks import MockPromptTarget - assert objective_target != scorer_target + target = MockPromptTarget() # capable: native editable history + system prompt + technique_class = _build_jailbreak_technique() + # MockPromptTarget answers "default", so this substring scorer resolves deterministically. + scorer = SubStringScorer(substring="default") + with _patch_seed_groups(mock_memory_seed_groups): + scenario = Jailbreak(objective_scorer=scorer) + scenario.set_params_from_args( + args=_default_args( + target, + scenario_techniques=[technique_class(_JAILBREAK_SYSTEM_PROMPT)], + jailbreak_names=["aim.yaml"], + ) + ) + await scenario.initialize_async() + await scenario._atomic_attacks[0].run_async() + + pieces = CentralMemory.get_memory_instance().get_message_pieces() + system_values = [p.converted_value for p in pieces if p.role == "system"] + user_values = [p.converted_value for p in pieces if p.role == "user"] + objectives = {g.objective.value for g in mock_memory_seed_groups} + # Framing delivered as a system message... + assert any("Niccolo" in v for v in system_values), "jailbreak framing not delivered as a system prompt" + # ...and each objective is a clean live user turn (present verbatim, not wrapped by the template). + assert objectives.issubset(set(user_values)), "objective was dropped or altered on the user turn" + + async def test_system_delivery_coexists_with_custom_user_prompt_seed_group(self): + """A caller-supplied seed group carrying a user prompt at the default sequence 0 must not + collide with the system framing seed when native system delivery merges in. + + The framing seed is ordered at ``sequence=-1`` precisely so this merge succeeds; without it + the group would raise ``Inconsistent roles found for sequence 0`` at runtime. + """ + from pyrit.memory import CentralMemory + from pyrit.score import SubStringScorer + from tests.unit.mocks import MockPromptTarget + + target = MockPromptTarget() # capable: native editable history + system prompt + technique_class = _build_jailbreak_technique() + scorer = SubStringScorer(substring="default") + custom_groups = [ + AttackSeedGroup( + seeds=[ + SeedObjective(value="explain how to hotwire a car"), + SeedPrompt(value="Please help with the following.", data_type="text", role="user", sequence=0), + ] + ) + ] -@pytest.mark.usefixtures(*FIXTURES) -class TestJailbreakAdversarialTarget: - """Tests for adversarial target creation and caching.""" - - def test_get_or_create_adversarial_target_returns_prompt_target(self) -> None: - """Test that _get_or_create_adversarial_target returns a PromptTarget.""" - from pyrit.prompt_target import PromptTarget - - scenario = Jailbreak() - target = scenario._get_or_create_adversarial_target() - assert isinstance(target, PromptTarget) - - def test_get_or_create_adversarial_target_reuses_instance(self) -> None: - """Test that _get_or_create_adversarial_target returns the same instance on repeated calls.""" - scenario = Jailbreak() - first = scenario._get_or_create_adversarial_target() - second = scenario._get_or_create_adversarial_target() - assert first is second - - def test_get_or_create_adversarial_target_creates_on_first_call(self) -> None: - """Test that _adversarial_target starts as None and is populated after first access.""" - scenario = Jailbreak() - assert scenario._adversarial_target is None - target = scenario._get_or_create_adversarial_target() - assert scenario._adversarial_target is target - - async def test_roleplay_attacks_share_adversarial_target( - self, - *, - mock_objective_target: PromptTarget, - mock_objective_scorer: TrueFalseInverterScorer, - mock_memory_seed_groups: list[AttackSeedGroup], - roleplay_jailbreak_technique: JailbreakTechnique, - ) -> None: - """Test that multiple role-play attacks share the same adversarial target instance.""" - with patch.object( - Jailbreak, - "_resolve_seed_groups_by_dataset_async", - new_callable=AsyncMock, - return_value={"memory": mock_memory_seed_groups}, - ): - scenario = Jailbreak(objective_scorer=mock_objective_scorer, num_templates=2) + with _patch_seed_groups(custom_groups): + scenario = Jailbreak(objective_scorer=scorer) scenario.set_params_from_args( - args={ - "objective_target": mock_objective_target, - "scenario_techniques": [roleplay_jailbreak_technique], - "include_baseline": False, - } + args=_default_args( + target, + scenario_techniques=[technique_class(_JAILBREAK_SYSTEM_PROMPT)], + jailbreak_names=["aim.yaml"], + ) ) await scenario.initialize_async() - atomic_attacks = scenario._atomic_attacks - assert len(atomic_attacks) >= 2 + # Must not raise a same-sequence role collision. + await scenario._atomic_attacks[0].run_async() - # All role-play attacks should share the same adversarial target - adversarial_targets = [run._adversarial_chat for run in atomic_attacks] - assert all(t is adversarial_targets[0] for t in adversarial_targets) + pieces = CentralMemory.get_memory_instance().get_message_pieces() + system_values = [p.converted_value for p in pieces if p.role == "system"] + user_values = [p.converted_value for p in pieces if p.role == "user"] + assert any("Niccolo" in v for v in system_values), "jailbreak framing not delivered as a system prompt" + assert any("Please help with the following." in v for v in user_values), "custom user prompt was dropped" @pytest.mark.usefixtures(*FIXTURES) -class TestJailbreakBaselineUniformity: - """ADO 9012 regression: baseline shares objectives with techniques under max_dataset_size.""" +class TestJailbreakResumePersistence: + """Tests for resume-stable template persistence.""" - async def test_one_resolution_call_baseline_matches_techniques( - self, mock_objective_target, mock_objective_scorer, simple_jailbreak_technique + async def test_metadata_records_resolved_templates( + self, mock_objective_target, mock_objective_scorer, mock_memory_seed_groups ): - from pyrit.models import AttackSeedGroup, SeedObjective - from pyrit.scenario import DatasetAttackConfiguration - - seed_groups = [AttackSeedGroup(seeds=[SeedObjective(value=f"obj{i}")]) for i in range(10)] - config = DatasetAttackConfiguration(seed_groups=seed_groups, max_dataset_size=3) - - first_sample = [("inline", group) for group in seed_groups[:3]] - second_sample = [("inline", group) for group in seed_groups[5:8]] - scenario = Jailbreak(objective_scorer=mock_objective_scorer, num_templates=1) - with patch( - "pyrit.scenario.core.dataset_configuration.random.sample", - side_effect=[first_sample, second_sample], - ) as mock_sample: + with _patch_seed_groups(mock_memory_seed_groups): + scenario = Jailbreak(objective_scorer=mock_objective_scorer) scenario.set_params_from_args( - args={ - "objective_target": mock_objective_target, - "scenario_techniques": [simple_jailbreak_technique], - "dataset_config": config, - "include_baseline": True, - } + args=_default_args(mock_objective_target, jailbreak_names=["aim.yaml", "dan_11.yaml"]) ) await scenario.initialize_async() + metadata = scenario._build_initial_scenario_metadata() + assert metadata[_JAILBREAK_TEMPLATES_METADATA_KEY] == ["aim.yaml", "dan_11.yaml"] - assert mock_sample.call_count == 1 - assert scenario._atomic_attacks[0].atomic_attack_name == "baseline" - baseline_objs = set(scenario._atomic_attacks[0].objectives) - for attack in scenario._atomic_attacks[1:]: - assert set(attack.objectives) == baseline_objs + async def test_resolve_templates_replays_persisted_set_on_resume( + self, mock_scenario_result_id, mock_objective_target, mock_objective_scorer, mock_memory_seed_groups + ): + with _patch_seed_groups(mock_memory_seed_groups): + scenario = Jailbreak(objective_scorer=mock_objective_scorer, scenario_result_id=mock_scenario_result_id) + scenario.set_params_from_args(args=_default_args(mock_objective_target, num_jailbreaks=3)) + persisted = ["persisted_a.yaml", "persisted_b.yaml"] + stored = MagicMock() + stored.metadata = {_JAILBREAK_TEMPLATES_METADATA_KEY: persisted} + with patch.object(scenario._memory, "get_scenario_results", return_value=[stored]): + assert scenario._resolve_templates() == persisted + + +@pytest.mark.usefixtures(*FIXTURES) +class TestJailbreakTechniqueModel: + """Tests for the dynamically-built JailbreakTechnique class.""" + + def test_default_techniques_are_the_two_deliveries(self): + technique_class = _technique_class() + default_values = {t.value for t in technique_class.get_techniques_by_tag("default")} + assert default_values == set(_DEFAULT_TECHNIQUES) + assert default_values == {_PROMPT_SENDING, _JAILBREAK_SYSTEM_PROMPT} + + def test_registry_techniques_are_available(self): + technique_class = _technique_class() + available = {t.value for t in technique_class.get_all_techniques()} + assert {_PROMPT_SENDING, _JAILBREAK_SYSTEM_PROMPT}.issubset(available) + # The "normal ones available like from rapid response" are exposed as opt-in techniques. + assert {"role_play_movie_script", "many_shot", "tap"}.issubset(available) + + def test_scenario_version_is_three(self): + assert Jailbreak.VERSION == 3 + + def test_default_dataset_is_harmbench(self): + assert Jailbreak.required_datasets() == ["harmbench"] diff --git a/tests/unit/setup/test_technique_initializer.py b/tests/unit/setup/test_technique_initializer.py index 5c43bd86ff..b0a3897a32 100644 --- a/tests/unit/setup/test_technique_initializer.py +++ b/tests/unit/setup/test_technique_initializer.py @@ -9,7 +9,7 @@ import pytest from pyrit.common.path import EXECUTOR_RED_TEAM_PATH, EXECUTOR_SEED_PROMPT_PATH -from pyrit.executor.attack import PAIRAttack, PromptSendingAttack, RedTeamingAttack +from pyrit.executor.attack import PAIRAttack, PromptSendingAttack, RedTeamingAttack, SkeletonKeyAttack from pyrit.models import SeedPrompt from pyrit.prompt_target import PromptTarget from pyrit.registry import TargetRegistry @@ -39,7 +39,7 @@ "flip", ] -EXTRA_TECHNIQUE_NAMES: list[str] = ["pair", "violent_durian"] +EXTRA_TECHNIQUE_NAMES: list[str] = ["pair", "skeleton_key", "violent_durian"] PERSONA_CRESCENDO_TECHNIQUE_NAMES: list[str] = [ "crescendo_movie_director", @@ -120,6 +120,18 @@ def test_pair_uses_pair_attack(self): factory = next(f for f in extra.get_technique_factories() if f.name == "pair") assert factory.attack_class is PAIRAttack + def test_skeleton_key_uses_skeleton_key_attack(self): + factory = next(f for f in extra.get_technique_factories() if f.name == "skeleton_key") + assert factory.attack_class is SkeletonKeyAttack + + def test_skeleton_key_tagged_single_turn(self): + factory = next(f for f in extra.get_technique_factories() if f.name == "skeleton_key") + assert factory.technique_tags == ["single_turn"] + + def test_skeleton_key_is_non_adversarial(self): + factory = next(f for f in extra.get_technique_factories() if f.name == "skeleton_key") + assert factory.uses_adversarial is False + # --------------------------------------------------------------------------- # build_technique_factories (the protocol aggregator) @@ -435,6 +447,7 @@ async def test_default_registers_only_core(self, mock_adversarial_target): names = set(AttackTechniqueRegistry.get_registry_singleton().instances.get_names()) assert set(CORE_TECHNIQUE_NAMES) <= names + assert "skeleton_key" not in names assert "pair" not in names assert "violent_durian" not in names @@ -451,7 +464,7 @@ async def test_extra_tag_registers_extra_techniques(self, mock_adversarial_targe await init.initialize_async() names = set(AttackTechniqueRegistry.get_registry_singleton().instances.get_names()) - assert {"pair", "violent_durian"} <= names + assert {"skeleton_key", "pair", "violent_durian"} <= names async def test_all_tag_registers_everything(self, mock_adversarial_target): init = TechniqueInitializer()