From 26f4b509fd02e2dbfcbc5d601b183188bb35ef6f Mon Sep 17 00:00:00 2001 From: Copilot <223556219+Copilot@users.noreply.github.com> Date: Tue, 14 Jul 2026 14:54:59 -0700 Subject: [PATCH 1/4] Fold GCG into pyrit/executor/promptgen; remove auxiliary_attacks GCGGenerator is already a PromptGeneratorStrategy (a peer of fuzzer and anecdoctor), so GCG belongs in the standard executor/promptgen tree rather than in its own quarantined auxiliary_attacks subtree. - Relocate the GCG package: pyrit/auxiliary_attacks/gcg/ -> pyrit/executor/promptgen/gcg/, rewriting all internal imports. - Delete the auxiliary_attacks package and its import-time ExperimentalWarning. - Keep torch strictly optional: the moved package retains its lazy PEP 562 __init__ and is deliberately not re-exported from promptgen/__init__.py, so base (no-torch) installs still import pyrit.executor.promptgen cleanly. - Move and retarget tests to tests/**/executor/promptgen/gcg/; drop the obsolete experimental-warning test; rename the notebook test to test_notebooks_gcg.py. - Move docs to doc/code/executor/gcg/ (0_gcg + 1_gcg_azure_ml), update myst.yml nav to nest them under executor, and drop the generated auxiliary_attacks API page (GCG folds into pyrit_executor_promptgen like fuzzer). - Update the AML entry point to python -m pyrit.executor.promptgen.gcg.experiments.run, the Dockerfile path, and all tooling configs (pyproject ty/ruff, MANIFEST.in, pre-commit, devcontainer). Hard removal of the old import path (no deprecation shim): the package was already flagged experimental. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 4c446ad2-7050-4b03-bd88-dfe83ec69434 --- .devcontainer/devcontainer.json | 2 +- .pre-commit-config.yaml | 2 +- MANIFEST.in | 2 +- .../gcg/0_gcg.ipynb} | 37 ++------- .../gcg/0_gcg.py} | 18 +---- .../gcg}/1_gcg_azure_ml.ipynb | 77 +++++++------------ .../gcg}/1_gcg_azure_ml.py | 24 ++---- doc/generate_docs/pct_to_ipynb.py | 2 +- doc/myst.yml | 7 +- pyproject.toml | 4 +- pyrit/auxiliary_attacks/__init__.py | 27 ------- pyrit/converter/suffix_append_converter.py | 2 +- .../promptgen}/gcg/__init__.py | 40 +++++----- .../promptgen}/gcg/attack/__init__.py | 0 .../promptgen}/gcg/attack/base/__init__.py | 0 .../gcg/attack/base/attack_manager.py | 2 +- .../promptgen}/gcg/attack/gcg/__init__.py | 0 .../promptgen}/gcg/attack/gcg/gcg_attack.py | 6 +- .../promptgen}/gcg/config.py | 8 +- .../promptgen}/gcg/data.py | 4 +- .../promptgen}/gcg/default_implementations.py | 10 +-- .../promptgen}/gcg/experiments/README.md | 8 +- .../promptgen}/gcg/experiments/__init__.py | 0 .../promptgen}/gcg/experiments/log.py | 0 .../promptgen}/gcg/experiments/run.py | 8 +- .../promptgen}/gcg/extension_protocols.py | 12 +-- .../promptgen}/gcg/generator.py | 20 ++--- .../promptgen}/gcg/src/Dockerfile | 0 .../end_to_end/auxiliary_attacks/__init__.py | 4 - .../promptgen/gcg}/test_gcg_aml_e2e.py | 4 +- .../test_gcg_attack_wiring_integration.py | 8 +- .../promptgen/gcg}/test_gcg_integration.py | 4 +- .../promptgen/gcg/test_notebooks_gcg.py} | 2 +- tests/unit/auxiliary_attacks/__init__.py | 2 - tests/unit/auxiliary_attacks/gcg/__init__.py | 2 - .../test_experimental_warning.py | 34 -------- .../gcg/test_attack_manager_helpers.py | 2 +- .../promptgen}/gcg/test_config.py | 2 +- .../promptgen}/gcg/test_data_and_config.py | 16 ++-- .../gcg/test_default_implementations.py | 26 +++---- .../gcg/test_extension_protocols.py | 12 +-- .../promptgen}/gcg/test_gcg_core.py | 4 +- .../promptgen}/gcg/test_generator.py | 6 +- .../gcg/test_get_goals_and_targets.py | 2 +- .../promptgen}/gcg/test_log.py | 12 +-- .../gcg/test_multi_prompt_attack.py | 2 +- .../promptgen}/gcg/test_public_api.py | 8 +- 47 files changed, 166 insertions(+), 308 deletions(-) rename doc/code/{auxiliary_attacks/0_auxiliary_attacks.ipynb => executor/gcg/0_gcg.ipynb} (93%) rename doc/code/{auxiliary_attacks/0_auxiliary_attacks.py => executor/gcg/0_gcg.py} (75%) rename doc/code/{auxiliary_attacks => executor/gcg}/1_gcg_azure_ml.ipynb (88%) rename doc/code/{auxiliary_attacks => executor/gcg}/1_gcg_azure_ml.py (89%) delete mode 100644 pyrit/auxiliary_attacks/__init__.py rename pyrit/{auxiliary_attacks => executor/promptgen}/gcg/__init__.py (61%) rename pyrit/{auxiliary_attacks => executor/promptgen}/gcg/attack/__init__.py (100%) rename pyrit/{auxiliary_attacks => executor/promptgen}/gcg/attack/base/__init__.py (100%) rename pyrit/{auxiliary_attacks => executor/promptgen}/gcg/attack/base/attack_manager.py (99%) rename pyrit/{auxiliary_attacks => executor/promptgen}/gcg/attack/gcg/__init__.py (100%) rename pyrit/{auxiliary_attacks => executor/promptgen}/gcg/attack/gcg/gcg_attack.py (98%) rename pyrit/{auxiliary_attacks => executor/promptgen}/gcg/config.py (98%) rename pyrit/{auxiliary_attacks => executor/promptgen}/gcg/data.py (92%) rename pyrit/{auxiliary_attacks => executor/promptgen}/gcg/default_implementations.py (97%) rename pyrit/{auxiliary_attacks => executor/promptgen}/gcg/experiments/README.md (81%) rename pyrit/{auxiliary_attacks => executor/promptgen}/gcg/experiments/__init__.py (100%) rename pyrit/{auxiliary_attacks => executor/promptgen}/gcg/experiments/log.py (100%) rename pyrit/{auxiliary_attacks => executor/promptgen}/gcg/experiments/run.py (93%) rename pyrit/{auxiliary_attacks => executor/promptgen}/gcg/extension_protocols.py (96%) rename pyrit/{auxiliary_attacks => executor/promptgen}/gcg/generator.py (98%) rename pyrit/{auxiliary_attacks => executor/promptgen}/gcg/src/Dockerfile (100%) delete mode 100644 tests/end_to_end/auxiliary_attacks/__init__.py rename tests/end_to_end/{auxiliary_attacks => executor/promptgen/gcg}/test_gcg_aml_e2e.py (96%) rename tests/integration/{auxiliary_attacks => executor/promptgen/gcg}/test_gcg_attack_wiring_integration.py (97%) rename tests/integration/{auxiliary_attacks => executor/promptgen/gcg}/test_gcg_integration.py (98%) rename tests/integration/{auxiliary_attacks/test_notebooks_auxiliary.py => executor/promptgen/gcg/test_notebooks_gcg.py} (90%) delete mode 100644 tests/unit/auxiliary_attacks/__init__.py delete mode 100644 tests/unit/auxiliary_attacks/gcg/__init__.py delete mode 100644 tests/unit/auxiliary_attacks/test_experimental_warning.py rename tests/unit/{auxiliary_attacks => executor/promptgen}/gcg/test_attack_manager_helpers.py (98%) rename tests/unit/{auxiliary_attacks => executor/promptgen}/gcg/test_config.py (99%) rename tests/unit/{auxiliary_attacks => executor/promptgen}/gcg/test_data_and_config.py (93%) rename tests/unit/{auxiliary_attacks => executor/promptgen}/gcg/test_default_implementations.py (94%) rename tests/unit/{auxiliary_attacks => executor/promptgen}/gcg/test_extension_protocols.py (92%) rename tests/unit/{auxiliary_attacks => executor/promptgen}/gcg/test_gcg_core.py (99%) rename tests/unit/{auxiliary_attacks => executor/promptgen}/gcg/test_generator.py (98%) rename tests/unit/{auxiliary_attacks => executor/promptgen}/gcg/test_get_goals_and_targets.py (98%) rename tests/unit/{auxiliary_attacks => executor/promptgen}/gcg/test_log.py (92%) rename tests/unit/{auxiliary_attacks => executor/promptgen}/gcg/test_multi_prompt_attack.py (98%) rename tests/unit/{auxiliary_attacks => executor/promptgen}/gcg/test_public_api.py (86%) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 025f9e4ccd..a4530bcc3d 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -47,7 +47,7 @@ "**/.pytest_cache/**": true, "**/build/**": true, "**/dist/**": true, - "**/pyrit/auxiliary_attacks/gcg/attack/**": true, + "**/pyrit/executor/promptgen/gcg/attack/**": true, "**/doc/**": true, "**/.ty_cache/**": true, "**/frontend/node_modules/**": true, diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 219f504b80..461321309a 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -91,5 +91,5 @@ repos: entry: uv run --link-mode=copy ty check language: system files: ^pyrit/ - exclude: ^pyrit/auxiliary_attacks/ + exclude: ^pyrit/executor/promptgen/gcg/ types: [python] diff --git a/MANIFEST.in b/MANIFEST.in index 5d5311b48e..9ceb63cd55 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -8,5 +8,5 @@ recursive-include pyrit *.png recursive-include pyrit *.wav recursive-include pyrit *.mp4 recursive-include pyrit *.md -include pyrit/auxiliary_attacks/gcg/src/Dockerfile +include pyrit/executor/promptgen/gcg/src/Dockerfile recursive-include pyrit/backend/frontend * diff --git a/doc/code/auxiliary_attacks/0_auxiliary_attacks.ipynb b/doc/code/executor/gcg/0_gcg.ipynb similarity index 93% rename from doc/code/auxiliary_attacks/0_auxiliary_attacks.ipynb rename to doc/code/executor/gcg/0_gcg.ipynb index e396a6b671..bd31eeba96 100644 --- a/doc/code/auxiliary_attacks/0_auxiliary_attacks.ipynb +++ b/doc/code/executor/gcg/0_gcg.ipynb @@ -7,7 +7,7 @@ "lines_to_next_cell": 0 }, "source": [ - "# Auxiliary Attacks" + "# Using GCG Adversarial Suffixes" ] }, { @@ -17,17 +17,7 @@ "lines_to_next_cell": 0 }, "source": [ - "> ⚠️ **Experimental module.** `pyrit.auxiliary_attacks` is experimental: its\n", - "> APIs may change in any release without a deprecation cycle. Importing the\n", - "> package (or any submodule) emits a `pyrit.exceptions.ExperimentalWarning`.\n", - "> Pin pyrit to a specific version if you depend on it. To silence the\n", - "> warning:\n", - ">\n", - "> ```python\n", - "> import warnings\n", - "> from pyrit.exceptions import ExperimentalWarning\n", - "> warnings.filterwarnings(\"ignore\", category=ExperimentalWarning)\n", - "> ```" + "The greedy coordinate gradient (GCG) [@zou2023gcg] algorithm generates adversarial suffixes that can be appended to a base prompt to jailbreak a language model. PyRIT provides an Azure Machine Learning (AML) pipeline for generating these suffixes." ] }, { @@ -36,25 +26,13 @@ "metadata": { "lines_to_next_cell": 0 }, - "source": [ - "Auxiliary attacks cover a variety of techniques that do not fit into the core PyRIT functionality.\n", - "\n", - "These attack pipelines may be useful to run before orchestrating other attacks. For example, we provide an Azure Machine Learning (AML) pipeline for generating suffixes using the greedy coordinate gradient (GCG) [@zou2023gcg] algorithm." - ] - }, - { - "cell_type": "markdown", - "id": "3", - "metadata": { - "lines_to_next_cell": 0 - }, "source": [ "## GCG Suffixes" ] }, { "cell_type": "markdown", - "id": "4", + "id": "3", "metadata": { "lines_to_next_cell": 0 }, @@ -67,7 +45,7 @@ { "cell_type": "code", "execution_count": null, - "id": "5", + "id": "4", "metadata": {}, "outputs": [ { @@ -167,7 +145,7 @@ }, { "cell_type": "markdown", - "id": "6", + "id": "5", "metadata": {}, "source": [ "Next, let's apply a GCG suffix trained on Phi-3-mini to the base prompt using the `SuffixAppendConverter`." @@ -176,7 +154,7 @@ { "cell_type": "code", "execution_count": null, - "id": "7", + "id": "6", "metadata": {}, "outputs": [ { @@ -264,7 +242,8 @@ ], "metadata": { "jupytext": { - "cell_metadata_filter": "-all" + "cell_metadata_filter": "-all", + "main_language": "python" }, "language_info": { "codemirror_mode": { diff --git a/doc/code/auxiliary_attacks/0_auxiliary_attacks.py b/doc/code/executor/gcg/0_gcg.py similarity index 75% rename from doc/code/auxiliary_attacks/0_auxiliary_attacks.py rename to doc/code/executor/gcg/0_gcg.py index 3f486c3f63..b90c880f9a 100644 --- a/doc/code/auxiliary_attacks/0_auxiliary_attacks.py +++ b/doc/code/executor/gcg/0_gcg.py @@ -9,23 +9,9 @@ # jupytext_version: 1.17.3 # --- # %% [markdown] -# # Auxiliary Attacks +# # Using GCG Adversarial Suffixes # %% [markdown] -# > ⚠️ **Experimental module.** `pyrit.auxiliary_attacks` is experimental: its -# > APIs may change in any release without a deprecation cycle. Importing the -# > package (or any submodule) emits a `pyrit.exceptions.ExperimentalWarning`. -# > Pin pyrit to a specific version if you depend on it. To silence the -# > warning: -# > -# > ```python -# > import warnings -# > from pyrit.exceptions import ExperimentalWarning -# > warnings.filterwarnings("ignore", category=ExperimentalWarning) -# > ``` -# %% [markdown] -# Auxiliary attacks cover a variety of techniques that do not fit into the core PyRIT functionality. -# -# These attack pipelines may be useful to run before orchestrating other attacks. For example, we provide an Azure Machine Learning (AML) pipeline for generating suffixes using the greedy coordinate gradient (GCG) [@zou2023gcg] algorithm. +# The greedy coordinate gradient (GCG) [@zou2023gcg] algorithm generates adversarial suffixes that can be appended to a base prompt to jailbreak a language model. PyRIT provides an Azure Machine Learning (AML) pipeline for generating these suffixes. # %% [markdown] # ## GCG Suffixes # %% [markdown] diff --git a/doc/code/auxiliary_attacks/1_gcg_azure_ml.ipynb b/doc/code/executor/gcg/1_gcg_azure_ml.ipynb similarity index 88% rename from doc/code/auxiliary_attacks/1_gcg_azure_ml.ipynb rename to doc/code/executor/gcg/1_gcg_azure_ml.ipynb index eb30d7e990..f2eeeda8a3 100644 --- a/doc/code/auxiliary_attacks/1_gcg_azure_ml.ipynb +++ b/doc/code/executor/gcg/1_gcg_azure_ml.ipynb @@ -12,23 +12,6 @@ "cell_type": "markdown", "id": "1", "metadata": {}, - "source": [ - "> ⚠️ **Experimental module.** `pyrit.auxiliary_attacks` is experimental: its\n", - "> APIs may change in any release without a deprecation cycle. Importing the\n", - "> package below emits a `pyrit.exceptions.ExperimentalWarning`. Pin pyrit to\n", - "> a specific version if you depend on it. To silence the warning:\n", - ">\n", - "> ```python\n", - "> import warnings\n", - "> from pyrit.exceptions import ExperimentalWarning\n", - "> warnings.filterwarnings(\"ignore\", category=ExperimentalWarning)\n", - "> ```" - ] - }, - { - "cell_type": "markdown", - "id": "2", - "metadata": {}, "source": [ "This notebook shows how to generate GCG [@zou2023gcg] suffixes using Azure Machine Learning (AML), which consists of three main steps:\n", "1. Connect to an Azure Machine Learning (AML) workspace.\n", @@ -38,7 +21,7 @@ }, { "cell_type": "markdown", - "id": "3", + "id": "2", "metadata": {}, "source": [ "## Connect to Azure Machine Learning Workspace" @@ -46,7 +29,7 @@ }, { "cell_type": "markdown", - "id": "4", + "id": "3", "metadata": {}, "source": [ "The [workspace](https://docs.microsoft.com/en-us/azure/machine-learning/concept-workspace) is the top-level resource for Azure Machine Learning (AML), providing a centralized place to work with all the artifacts you create when using AML. In this section, we will connect to the workspace in which the job will be run.\n", @@ -57,7 +40,7 @@ { "cell_type": "code", "execution_count": null, - "id": "5", + "id": "4", "metadata": {}, "outputs": [ { @@ -86,7 +69,7 @@ }, { "cell_type": "markdown", - "id": "6", + "id": "5", "metadata": {}, "source": [ "The Azure ML SDK emits a fair amount of telemetry to stderr that looks\n", @@ -101,7 +84,7 @@ { "cell_type": "code", "execution_count": null, - "id": "7", + "id": "6", "metadata": {}, "outputs": [], "source": [ @@ -115,7 +98,7 @@ { "cell_type": "code", "execution_count": null, - "id": "8", + "id": "7", "metadata": {}, "outputs": [ { @@ -135,7 +118,7 @@ }, { "cell_type": "markdown", - "id": "9", + "id": "8", "metadata": {}, "source": [ "## Create AML Environment" @@ -143,24 +126,24 @@ }, { "cell_type": "markdown", - "id": "10", + "id": "9", "metadata": {}, "source": [ "To install the dependencies needed to run GCG, we create an AML environment from a\n", - "[Dockerfile](../../../pyrit/auxiliary_attacks/gcg/src/Dockerfile). The Dockerfile uses\n", + "[Dockerfile](../../../../pyrit/executor/promptgen/gcg/src/Dockerfile). The Dockerfile uses\n", "an NVIDIA CUDA base image with Python 3.11 and installs PyRIT with the `gcg` extra." ] }, { "cell_type": "code", "execution_count": null, - "id": "11", + "id": "10", "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "Environment({'arm_type': 'environment_version', 'latest_version': None, 'image': None, 'intellectual_property': None, 'is_anonymous': False, 'auto_increment_version': False, 'auto_delete_setting': None, 'name': 'pyrit-gcg', 'description': 'PyRIT GCG environment: CUDA 12.1 + Python 3.11 + pip install -e .[gcg]', 'tags': {'Owner': 'unknown'}, 'properties': {'azureml.labels': 'latest'}, 'print_as_yaml': False, 'id': '/subscriptions/db1ba766-2ca3-42c6-a19a-0f0d43134a8c/resourceGroups/gcg-romanlutz/providers/Microsoft.MachineLearningServices/workspaces/gcg-romanlutz/environments/pyrit-gcg/versions/17', 'Resource__source_path': '', 'base_path': './git/copilot-worktrees/PyRIT/romanlutz-upgraded-barnacle/doc/code/auxiliary_attacks', 'creation_context': , 'serialize': , 'version': '17', 'conda_file': None, 'build': , 'inference_config': None, 'os_type': 'Linux', 'conda_file_path': None, 'path': None, 'datastore': None, 'upload_hash': None, 'translated_conda_file': None})" + "Environment({'arm_type': 'environment_version', 'latest_version': None, 'image': None, 'intellectual_property': None, 'is_anonymous': False, 'auto_increment_version': False, 'auto_delete_setting': None, 'name': 'pyrit-gcg', 'description': 'PyRIT GCG environment: CUDA 12.1 + Python 3.11 + pip install -e .[gcg]', 'tags': {'Owner': 'unknown'}, 'properties': {'azureml.labels': 'latest'}, 'print_as_yaml': False, 'id': '/subscriptions/db1ba766-2ca3-42c6-a19a-0f0d43134a8c/resourceGroups/gcg-romanlutz/providers/Microsoft.MachineLearningServices/workspaces/gcg-romanlutz/environments/pyrit-gcg/versions/17', 'Resource__source_path': '', 'base_path': './git/copilot-worktrees/PyRIT/romanlutz-upgraded-barnacle/doc/code/executor/gcg', 'creation_context': , 'serialize': , 'version': '17', 'conda_file': None, 'build': , 'inference_config': None, 'os_type': 'Linux', 'conda_file_path': None, 'path': None, 'datastore': None, 'upload_hash': None, 'translated_conda_file': None})" ] }, "execution_count": null, @@ -180,7 +163,7 @@ "env_docker_context = Environment(\n", " build=BuildContext(\n", " path=Path(HOME_PATH),\n", - " dockerfile_path=\"pyrit/auxiliary_attacks/gcg/src/Dockerfile\",\n", + " dockerfile_path=\"pyrit/executor/promptgen/gcg/src/Dockerfile\",\n", " ),\n", " name=\"pyrit-gcg\",\n", " description=\"PyRIT GCG environment: CUDA 12.1 + Python 3.11 + pip install -e .[gcg]\",\n", @@ -192,7 +175,7 @@ }, { "cell_type": "markdown", - "id": "12", + "id": "11", "metadata": {}, "source": [ "## Submit Training Job to AML" @@ -200,11 +183,11 @@ }, { "cell_type": "markdown", - "id": "13", + "id": "12", "metadata": {}, "source": [ "Finally, we configure the command to run the GCG algorithm. The entry point is\n", - "[`pyrit.auxiliary_attacks.gcg.experiments.run`](../../../pyrit/auxiliary_attacks/gcg/experiments/run.py),\n", + "[`pyrit.executor.promptgen.gcg.experiments.run`](../../../../pyrit/executor/promptgen/gcg/experiments/run.py),\n", "invoked as a module so the uploaded code snapshot takes priority over the\n", "Docker-installed package (Python's `-m` flag puts the cwd at the front of `sys.path`).\n", "\n", @@ -212,7 +195,7 @@ "``GCGDataConfig`` (CSV paths/counts). We build both locally with whatever\n", "overrides we want, serialize each into a JSON file the AML job can read as\n", "an input, and ship those paths through the job command. Defaults come from\n", - "the dataclasses in ``pyrit.auxiliary_attacks.gcg.config``; goals and targets\n", + "the dataclasses in ``pyrit.executor.promptgen.gcg.config``; goals and targets\n", "flow into ``GCGGenerator.execute_async`` at runtime, not through the config.\n", "\n", "We also have to specify a GPU compute target. In our experience, a GPU instance with\n", @@ -226,22 +209,13 @@ { "cell_type": "code", "execution_count": null, - "id": "14", + "id": "13", "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "./AppData/Local/Temp/ipykernel_95196/4292719205.py:3: ExperimentalWarning: pyrit.auxiliary_attacks is experimental: APIs may change in any release without a deprecation cycle. Pin pyrit to a specific version if you depend on this module. To silence: warnings.filterwarnings('ignore', category=pyrit.exceptions.ExperimentalWarning).\n", - " from pyrit.auxiliary_attacks.gcg import (\n" - ] - } - ], + "outputs": [], "source": [ "import tempfile\n", "\n", - "from pyrit.auxiliary_attacks.gcg import (\n", + "from pyrit.executor.promptgen.gcg import (\n", " GCGAlgorithmConfig,\n", " GCGConfig,\n", " GCGDataConfig,\n", @@ -271,7 +245,7 @@ { "cell_type": "code", "execution_count": null, - "id": "15", + "id": "14", "metadata": {}, "outputs": [], "source": [ @@ -280,7 +254,7 @@ "job = command(\n", " code=Path(HOME_PATH),\n", " command=(\n", - " \"python -m pyrit.auxiliary_attacks.gcg.experiments.run\"\n", + " \"python -m pyrit.executor.promptgen.gcg.experiments.run\"\n", " \" --config ${{inputs.config}}\"\n", " \" --data ${{inputs.data}}\"\n", " \" --output-dir ${{outputs.results}}\"\n", @@ -302,7 +276,7 @@ { "cell_type": "code", "execution_count": null, - "id": "16", + "id": "15", "metadata": {}, "outputs": [ { @@ -373,7 +347,7 @@ }, { "cell_type": "markdown", - "id": "17", + "id": "16", "metadata": {}, "source": [ "## Wait for the Job to Complete and Inspect the Generated Suffix\n", @@ -394,7 +368,7 @@ { "cell_type": "code", "execution_count": null, - "id": "18", + "id": "17", "metadata": {}, "outputs": [ { @@ -490,7 +464,8 @@ ], "metadata": { "jupytext": { - "cell_metadata_filter": "-all" + "cell_metadata_filter": "-all", + "main_language": "python" }, "language_info": { "codemirror_mode": { diff --git a/doc/code/auxiliary_attacks/1_gcg_azure_ml.py b/doc/code/executor/gcg/1_gcg_azure_ml.py similarity index 89% rename from doc/code/auxiliary_attacks/1_gcg_azure_ml.py rename to doc/code/executor/gcg/1_gcg_azure_ml.py index b6620f65b3..a229f1ce14 100644 --- a/doc/code/auxiliary_attacks/1_gcg_azure_ml.py +++ b/doc/code/executor/gcg/1_gcg_azure_ml.py @@ -12,18 +12,6 @@ # %% [markdown] # # 1. Generating GCG Suffixes Using Azure Machine Learning -# %% [markdown] -# > ⚠️ **Experimental module.** `pyrit.auxiliary_attacks` is experimental: its -# > APIs may change in any release without a deprecation cycle. Importing the -# > package below emits a `pyrit.exceptions.ExperimentalWarning`. Pin pyrit to -# > a specific version if you depend on it. To silence the warning: -# > -# > ```python -# > import warnings -# > from pyrit.exceptions import ExperimentalWarning -# > warnings.filterwarnings("ignore", category=ExperimentalWarning) -# > ``` - # %% [markdown] # This notebook shows how to generate GCG [@zou2023gcg] suffixes using Azure Machine Learning (AML), which consists of three main steps: # 1. Connect to an Azure Machine Learning (AML) workspace. @@ -77,7 +65,7 @@ # %% [markdown] # To install the dependencies needed to run GCG, we create an AML environment from a -# [Dockerfile](../../../pyrit/auxiliary_attacks/gcg/src/Dockerfile). The Dockerfile uses +# [Dockerfile](../../../../pyrit/executor/promptgen/gcg/src/Dockerfile). The Dockerfile uses # an NVIDIA CUDA base image with Python 3.11 and installs PyRIT with the `gcg` extra. # %% @@ -92,7 +80,7 @@ env_docker_context = Environment( build=BuildContext( path=Path(HOME_PATH), - dockerfile_path="pyrit/auxiliary_attacks/gcg/src/Dockerfile", + dockerfile_path="pyrit/executor/promptgen/gcg/src/Dockerfile", ), name="pyrit-gcg", description="PyRIT GCG environment: CUDA 12.1 + Python 3.11 + pip install -e .[gcg]", @@ -106,7 +94,7 @@ # %% [markdown] # Finally, we configure the command to run the GCG algorithm. The entry point is -# [`pyrit.auxiliary_attacks.gcg.experiments.run`](../../../pyrit/auxiliary_attacks/gcg/experiments/run.py), +# [`pyrit.executor.promptgen.gcg.experiments.run`](../../../../pyrit/executor/promptgen/gcg/experiments/run.py), # invoked as a module so the uploaded code snapshot takes priority over the # Docker-installed package (Python's `-m` flag puts the cwd at the front of `sys.path`). # @@ -114,7 +102,7 @@ # ``GCGDataConfig`` (CSV paths/counts). We build both locally with whatever # overrides we want, serialize each into a JSON file the AML job can read as # an input, and ship those paths through the job command. Defaults come from -# the dataclasses in ``pyrit.auxiliary_attacks.gcg.config``; goals and targets +# the dataclasses in ``pyrit.executor.promptgen.gcg.config``; goals and targets # flow into ``GCGGenerator.execute_async`` at runtime, not through the config. # # We also have to specify a GPU compute target. In our experience, a GPU instance with @@ -127,7 +115,7 @@ # %% import tempfile -from pyrit.auxiliary_attacks.gcg import ( +from pyrit.executor.promptgen.gcg import ( GCGAlgorithmConfig, GCGConfig, GCGDataConfig, @@ -159,7 +147,7 @@ job = command( code=Path(HOME_PATH), command=( - "python -m pyrit.auxiliary_attacks.gcg.experiments.run" + "python -m pyrit.executor.promptgen.gcg.experiments.run" " --config ${{inputs.config}}" " --data ${{inputs.data}}" " --output-dir ${{outputs.results}}" diff --git a/doc/generate_docs/pct_to_ipynb.py b/doc/generate_docs/pct_to_ipynb.py index 68270cfe95..e3ef4cacc0 100644 --- a/doc/generate_docs/pct_to_ipynb.py +++ b/doc/generate_docs/pct_to_ipynb.py @@ -8,7 +8,7 @@ skip_files = { "conf.py", - # auxiliary_attacks + # executor / gcg "1_gcg_azure_ml.py", # missing required env variables # converters "2_audio_converters.py", # requires Azure Speech API key diff --git a/doc/myst.yml b/doc/myst.yml index c0f22d68ff..c302b62f5a 100644 --- a/doc/myst.yml +++ b/doc/myst.yml @@ -108,6 +108,9 @@ project: - file: code/executor/6_benchmark.ipynb - file: code/executor/7_promptgen.ipynb - file: code/executor/8_modality_feedback.ipynb + - file: code/executor/gcg/0_gcg.ipynb + children: + - file: code/executor/gcg/1_gcg_azure_ml.ipynb - file: code/targets/0_prompt_targets.md children: - file: code/targets/1_openai_chat_target.ipynb @@ -162,9 +165,6 @@ project: - file: code/setup/2_resiliency.ipynb - file: code/setup/default_values.md - file: code/setup/pyrit_initializer.ipynb - - file: code/auxiliary_attacks/0_auxiliary_attacks.ipynb - children: - - file: code/auxiliary_attacks/1_gcg_azure_ml.ipynb - file: code/scenarios/0_scenarios.ipynb children: - file: code/scenarios/0_attack_techniques.ipynb @@ -180,7 +180,6 @@ project: children: - file: api/pyrit_analytics.md - file: api/pyrit_auth.md - - file: api/pyrit_auxiliary_attacks.md - file: api/pyrit_cli_api_client.md - file: api/pyrit_cli_pyrit_scan.md - file: api/pyrit_cli_pyrit_shell.md diff --git a/pyproject.toml b/pyproject.toml index a4a88c776f..5561171afe 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -211,7 +211,7 @@ replace-imports-with-any = [ ] [tool.ty.src] -exclude = ["doc/code/", "pyrit/auxiliary_attacks/"] +exclude = ["doc/code/", "pyrit/executor/promptgen/gcg/"] # Relax type checking for HuggingFace module [[tool.ty.overrides]] @@ -426,7 +426,7 @@ notice-rgx = "Copyright \\(c\\) Microsoft Corporation\\.\\s*\\n.*Licensed under # Backend API routes raise HTTPException handled by FastAPI, not true exceptions "pyrit/backend/**/*.py" = ["DOC501", "B008"] # Vendored GCG attack code — external project with its own style -"pyrit/auxiliary_attacks/**/*.py" = ["A", "B905", "D", "DOC", "N", "PERF", "SIM101", "SIM108"] +"pyrit/executor/promptgen/gcg/**/*.py" = ["A", "B905", "D", "DOC", "N", "PERF", "SIM101", "SIM108"] "pyrit/__init__.py" = ["D104"] # Allow broad pytest.raises(Exception) in tests "tests/**/*.py" = ["ANN", "B017"] diff --git a/pyrit/auxiliary_attacks/__init__.py b/pyrit/auxiliary_attacks/__init__.py deleted file mode 100644 index aadde75182..0000000000 --- a/pyrit/auxiliary_attacks/__init__.py +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright (c) Microsoft Corporation. -# Licensed under the MIT license. - -""" -Experimental auxiliary attack implementations (e.g., GCG). - -This subpackage is **experimental**: APIs may change in any release without a -deprecation cycle. Pin pyrit to a specific version if you depend on it. To -silence the warning emitted on import:: - - import warnings - from pyrit.exceptions import ExperimentalWarning - warnings.filterwarnings("ignore", category=ExperimentalWarning) -""" - -import warnings - -from pyrit.exceptions import ExperimentalWarning - -warnings.warn( - "pyrit.auxiliary_attacks is experimental: APIs may change in any release " - "without a deprecation cycle. Pin pyrit to a specific version if you depend " - "on this module. To silence: " - "warnings.filterwarnings('ignore', category=pyrit.exceptions.ExperimentalWarning).", - ExperimentalWarning, - stacklevel=2, -) diff --git a/pyrit/converter/suffix_append_converter.py b/pyrit/converter/suffix_append_converter.py index effb23890f..14120e6a41 100644 --- a/pyrit/converter/suffix_append_converter.py +++ b/pyrit/converter/suffix_append_converter.py @@ -11,7 +11,7 @@ class SuffixAppendConverter(Converter): Appends a specified suffix to the prompt. E.g. with a suffix `!!!`, it converts a prompt of `test` to `test !!!`. - See https://github.com/microsoft/PyRIT/tree/main/pyrit/auxiliary_attacks/gcg for adversarial suffix generation. + See https://github.com/microsoft/PyRIT/tree/main/pyrit/executor/promptgen/gcg for adversarial suffix generation. """ SUPPORTED_INPUT_TYPES = ("text",) diff --git a/pyrit/auxiliary_attacks/gcg/__init__.py b/pyrit/executor/promptgen/gcg/__init__.py similarity index 61% rename from pyrit/auxiliary_attacks/gcg/__init__.py rename to pyrit/executor/promptgen/gcg/__init__.py index 160b2f313a..3becd552ab 100644 --- a/pyrit/auxiliary_attacks/gcg/__init__.py +++ b/pyrit/executor/promptgen/gcg/__init__.py @@ -9,7 +9,7 @@ Example: - from pyrit.auxiliary_attacks.gcg import ( + from pyrit.executor.promptgen.gcg import ( GCG, GCGAlgorithmConfig, GCGModelConfig, @@ -27,7 +27,7 @@ from typing import TYPE_CHECKING, Any -from pyrit.auxiliary_attacks.gcg.config import ( +from pyrit.executor.promptgen.gcg.config import ( GCGAlgorithmConfig, GCGConfig, GCGDataConfig, @@ -37,7 +37,7 @@ ) # Torch-dependent symbols are exposed lazily via PEP 562 __getattr__ so that -# `from pyrit.auxiliary_attacks.gcg import GCGConfig` works on installs that +# `from pyrit.executor.promptgen.gcg import GCGConfig` works on installs that # only have the base `dev` extra (no torch). Touching any of these names from # the package root triggers the underlying module import on first access; if # torch is missing the user gets a clear ModuleNotFoundError pointing at torch. @@ -46,36 +46,36 @@ # module imports cleanly without torch) but are routed through the same lazy # mechanism so all GCG public symbols share one re-export pathway. _LAZY_IMPORTS = { - "CandidateFilter": ("pyrit.auxiliary_attacks.gcg.extension_protocols", "CandidateFilter"), - "CrossEntropyLoss": ("pyrit.auxiliary_attacks.gcg.default_implementations", "CrossEntropyLoss"), - "GCG": ("pyrit.auxiliary_attacks.gcg.generator", "GCGGenerator"), - "GCGContext": ("pyrit.auxiliary_attacks.gcg.generator", "GCGContext"), - "GCGGenerator": ("pyrit.auxiliary_attacks.gcg.generator", "GCGGenerator"), - "GCGResult": ("pyrit.auxiliary_attacks.gcg.generator", "GCGResult"), - "LengthPreservingFilter": ("pyrit.auxiliary_attacks.gcg.default_implementations", "LengthPreservingFilter"), - "LiteralStringInit": ("pyrit.auxiliary_attacks.gcg.default_implementations", "LiteralStringInit"), - "LossFunction": ("pyrit.auxiliary_attacks.gcg.extension_protocols", "LossFunction"), - "SamplingStrategy": ("pyrit.auxiliary_attacks.gcg.extension_protocols", "SamplingStrategy"), - "StandardGCGSampling": ("pyrit.auxiliary_attacks.gcg.default_implementations", "StandardGCGSampling"), - "SuffixInitializer": ("pyrit.auxiliary_attacks.gcg.extension_protocols", "SuffixInitializer"), - "load_goals_and_targets": ("pyrit.auxiliary_attacks.gcg.data", "load_goals_and_targets"), + "CandidateFilter": ("pyrit.executor.promptgen.gcg.extension_protocols", "CandidateFilter"), + "CrossEntropyLoss": ("pyrit.executor.promptgen.gcg.default_implementations", "CrossEntropyLoss"), + "GCG": ("pyrit.executor.promptgen.gcg.generator", "GCGGenerator"), + "GCGContext": ("pyrit.executor.promptgen.gcg.generator", "GCGContext"), + "GCGGenerator": ("pyrit.executor.promptgen.gcg.generator", "GCGGenerator"), + "GCGResult": ("pyrit.executor.promptgen.gcg.generator", "GCGResult"), + "LengthPreservingFilter": ("pyrit.executor.promptgen.gcg.default_implementations", "LengthPreservingFilter"), + "LiteralStringInit": ("pyrit.executor.promptgen.gcg.default_implementations", "LiteralStringInit"), + "LossFunction": ("pyrit.executor.promptgen.gcg.extension_protocols", "LossFunction"), + "SamplingStrategy": ("pyrit.executor.promptgen.gcg.extension_protocols", "SamplingStrategy"), + "StandardGCGSampling": ("pyrit.executor.promptgen.gcg.default_implementations", "StandardGCGSampling"), + "SuffixInitializer": ("pyrit.executor.promptgen.gcg.extension_protocols", "SuffixInitializer"), + "load_goals_and_targets": ("pyrit.executor.promptgen.gcg.data", "load_goals_and_targets"), } if TYPE_CHECKING: - from pyrit.auxiliary_attacks.gcg.data import load_goals_and_targets - from pyrit.auxiliary_attacks.gcg.default_implementations import ( + from pyrit.executor.promptgen.gcg.data import load_goals_and_targets + from pyrit.executor.promptgen.gcg.default_implementations import ( CrossEntropyLoss, LengthPreservingFilter, LiteralStringInit, StandardGCGSampling, ) - from pyrit.auxiliary_attacks.gcg.extension_protocols import ( + from pyrit.executor.promptgen.gcg.extension_protocols import ( CandidateFilter, LossFunction, SamplingStrategy, SuffixInitializer, ) - from pyrit.auxiliary_attacks.gcg.generator import ( + from pyrit.executor.promptgen.gcg.generator import ( GCGContext, GCGGenerator, GCGResult, diff --git a/pyrit/auxiliary_attacks/gcg/attack/__init__.py b/pyrit/executor/promptgen/gcg/attack/__init__.py similarity index 100% rename from pyrit/auxiliary_attacks/gcg/attack/__init__.py rename to pyrit/executor/promptgen/gcg/attack/__init__.py diff --git a/pyrit/auxiliary_attacks/gcg/attack/base/__init__.py b/pyrit/executor/promptgen/gcg/attack/base/__init__.py similarity index 100% rename from pyrit/auxiliary_attacks/gcg/attack/base/__init__.py rename to pyrit/executor/promptgen/gcg/attack/base/__init__.py diff --git a/pyrit/auxiliary_attacks/gcg/attack/base/attack_manager.py b/pyrit/executor/promptgen/gcg/attack/base/attack_manager.py similarity index 99% rename from pyrit/auxiliary_attacks/gcg/attack/base/attack_manager.py rename to pyrit/executor/promptgen/gcg/attack/base/attack_manager.py index ef61ebc4d7..c5c65c07f9 100644 --- a/pyrit/auxiliary_attacks/gcg/attack/base/attack_manager.py +++ b/pyrit/executor/promptgen/gcg/attack/base/attack_manager.py @@ -29,7 +29,7 @@ Phi3ForCausalLM, ) -from pyrit.auxiliary_attacks.gcg.experiments.log import ( +from pyrit.executor.promptgen.gcg.experiments.log import ( log_gpu_memory, log_loss, log_table_summary, diff --git a/pyrit/auxiliary_attacks/gcg/attack/gcg/__init__.py b/pyrit/executor/promptgen/gcg/attack/gcg/__init__.py similarity index 100% rename from pyrit/auxiliary_attacks/gcg/attack/gcg/__init__.py rename to pyrit/executor/promptgen/gcg/attack/gcg/__init__.py diff --git a/pyrit/auxiliary_attacks/gcg/attack/gcg/gcg_attack.py b/pyrit/executor/promptgen/gcg/attack/gcg/gcg_attack.py similarity index 98% rename from pyrit/auxiliary_attacks/gcg/attack/gcg/gcg_attack.py rename to pyrit/executor/promptgen/gcg/attack/gcg/gcg_attack.py index ea0677527e..dfe5a11d31 100644 --- a/pyrit/auxiliary_attacks/gcg/attack/gcg/gcg_attack.py +++ b/pyrit/executor/promptgen/gcg/attack/gcg/gcg_attack.py @@ -10,19 +10,19 @@ import torch.nn as nn from tqdm.auto import tqdm -from pyrit.auxiliary_attacks.gcg.attack.base.attack_manager import ( +from pyrit.executor.promptgen.gcg.attack.base.attack_manager import ( AttackPrompt, MultiPromptAttack, PromptManager, get_embedding_matrix, get_embeddings, ) -from pyrit.auxiliary_attacks.gcg.default_implementations import ( +from pyrit.executor.promptgen.gcg.default_implementations import ( CrossEntropyLoss, LengthPreservingFilter, StandardGCGSampling, ) -from pyrit.auxiliary_attacks.gcg.extension_protocols import CandidateFilter, LossFunction, SamplingStrategy +from pyrit.executor.promptgen.gcg.extension_protocols import CandidateFilter, LossFunction, SamplingStrategy logger = logging.getLogger(__name__) diff --git a/pyrit/auxiliary_attacks/gcg/config.py b/pyrit/executor/promptgen/gcg/config.py similarity index 98% rename from pyrit/auxiliary_attacks/gcg/config.py rename to pyrit/executor/promptgen/gcg/config.py index c2debada6e..ad07b1122c 100644 --- a/pyrit/auxiliary_attacks/gcg/config.py +++ b/pyrit/executor/promptgen/gcg/config.py @@ -25,7 +25,7 @@ if TYPE_CHECKING: from pathlib import Path - from pyrit.auxiliary_attacks.gcg.extension_protocols import ( + from pyrit.executor.promptgen.gcg.extension_protocols import ( CandidateFilter, LossFunction, SamplingStrategy, @@ -78,7 +78,7 @@ class GCGDataConfig: Used as a typed bundle for AML transport (a job ships its data config as a separate JSON file alongside the strategy ``GCGConfig``). Library callers loading goals/targets from a CSV can construct one and pass it to - ``pyrit.auxiliary_attacks.gcg.data.load_goals_and_targets``. + ``pyrit.executor.promptgen.gcg.data.load_goals_and_targets``. Attributes: train_data (str): URL or filesystem path to the training-data CSV. Empty @@ -209,7 +209,7 @@ def __post_init__(self) -> None: self._validate_extensions() def _validate_extensions(self) -> None: - from pyrit.auxiliary_attacks.gcg.extension_protocols import ( + from pyrit.executor.promptgen.gcg.extension_protocols import ( CandidateFilter, LossFunction, SamplingStrategy, @@ -284,7 +284,7 @@ class GCGOutputConfig: class GCGConfig: """Top-level strategy configuration for one GCG attack run. - Bundles everything ``pyrit.auxiliary_attacks.gcg.GCGGenerator``'s + Bundles everything ``pyrit.executor.promptgen.gcg.GCGGenerator``'s constructor needs. Per-execution data (goals, targets) is **not** here — those flow through ``GCGGenerator.execute_async``, and for AML transport they ride alongside this object as a separate ``GCGDataConfig`` JSON. diff --git a/pyrit/auxiliary_attacks/gcg/data.py b/pyrit/executor/promptgen/gcg/data.py similarity index 92% rename from pyrit/auxiliary_attacks/gcg/data.py rename to pyrit/executor/promptgen/gcg/data.py index 1c8b8efd3a..84ca1e79f1 100644 --- a/pyrit/auxiliary_attacks/gcg/data.py +++ b/pyrit/executor/promptgen/gcg/data.py @@ -13,12 +13,12 @@ from types import SimpleNamespace from typing import TYPE_CHECKING -from pyrit.auxiliary_attacks.gcg.attack.base.attack_manager import ( +from pyrit.executor.promptgen.gcg.attack.base.attack_manager import ( get_goals_and_targets as _legacy_loader, ) if TYPE_CHECKING: - from pyrit.auxiliary_attacks.gcg.config import GCGDataConfig + from pyrit.executor.promptgen.gcg.config import GCGDataConfig def load_goals_and_targets( diff --git a/pyrit/auxiliary_attacks/gcg/default_implementations.py b/pyrit/executor/promptgen/gcg/default_implementations.py similarity index 97% rename from pyrit/auxiliary_attacks/gcg/default_implementations.py rename to pyrit/executor/promptgen/gcg/default_implementations.py index 3967c128c7..502c6f6065 100644 --- a/pyrit/auxiliary_attacks/gcg/default_implementations.py +++ b/pyrit/executor/promptgen/gcg/default_implementations.py @@ -42,7 +42,7 @@ class StandardGCGSampling: sampler, which always samples uniformly within the top-k. Reproduces ``GCGPromptManager.sample_control`` from - ``pyrit/auxiliary_attacks/gcg/attack/gcg/gcg_attack.py`` byte-for-byte. + ``pyrit/executor/promptgen/gcg/attack/gcg/gcg_attack.py`` byte-for-byte. """ def sample_candidates( @@ -118,9 +118,9 @@ class CrossEntropyLoss: The same skip is applied when ``target_weight == 0`` for symmetry. Reproduces ``AttackPrompt.target_loss`` + ``AttackPrompt.control_loss`` - from ``pyrit/auxiliary_attacks/gcg/attack/base/attack_manager.py``, + from ``pyrit/executor/promptgen/gcg/attack/base/attack_manager.py``, combined per ``GCGMultiPromptAttack.step`` in - ``pyrit/auxiliary_attacks/gcg/attack/gcg/gcg_attack.py``. + ``pyrit/executor/promptgen/gcg/attack/gcg/gcg_attack.py``. """ def __init__(self, *, target_weight: float = 1.0, control_weight: float = 0.0) -> None: @@ -217,7 +217,7 @@ class LengthPreservingFilter: matching the safety pass at the top of ``get_filtered_cands``. Reproduces ``MultiPromptAttack.get_filtered_cands`` from - ``pyrit/auxiliary_attacks/gcg/attack/base/attack_manager.py``. + ``pyrit/executor/promptgen/gcg/attack/base/attack_manager.py``. """ def __init__(self, *, filter: bool = True) -> None: @@ -292,7 +292,7 @@ class LiteralStringInit: Reproduces the literal-string ``control_init`` parameter assignment (``self.control = control_init``) inside ``AttackPrompt.__init__`` in - ``pyrit/auxiliary_attacks/gcg/attack/base/attack_manager.py``. + ``pyrit/executor/promptgen/gcg/attack/base/attack_manager.py``. """ def __init__(self, *, suffix: str) -> None: diff --git a/pyrit/auxiliary_attacks/gcg/experiments/README.md b/pyrit/executor/promptgen/gcg/experiments/README.md similarity index 81% rename from pyrit/auxiliary_attacks/gcg/experiments/README.md rename to pyrit/executor/promptgen/gcg/experiments/README.md index bade583b7a..0409d33616 100644 --- a/pyrit/auxiliary_attacks/gcg/experiments/README.md +++ b/pyrit/executor/promptgen/gcg/experiments/README.md @@ -11,7 +11,7 @@ The primary entry point is `GCG.execute_async` (`GCG` is an alias for ```python import asyncio -from pyrit.auxiliary_attacks.gcg import GCG, GCGModelConfig +from pyrit.executor.promptgen.gcg import GCG, GCGModelConfig generator = GCG( models=[GCGModelConfig(name="meta-llama/Llama-2-7b-chat-hf")], @@ -21,7 +21,7 @@ result = asyncio.run(generator.execute_async(goals=[...], targets=[...])) `GCGConfig` is composed of nested sub-configs (`GCGModelConfig`, `GCGDataConfig`, `GCGAlgorithmConfig`, `GCGStrategyConfig`, `GCGOutputConfig`); all are re-exported -from `pyrit.auxiliary_attacks.gcg`. See `pyrit/auxiliary_attacks/gcg/config.py` +from `pyrit.executor.promptgen.gcg`. See `pyrit/executor/promptgen/gcg/config.py` for the full surface and defaults. ### Running on Azure ML @@ -34,10 +34,10 @@ config.to_json_file("inputs/config.json") ``` ``` -python -m pyrit.auxiliary_attacks.gcg.experiments.run --config inputs/config.json +python -m pyrit.executor.promptgen.gcg.experiments.run --config inputs/config.json ``` -The notebook at `doc/code/auxiliary_attacks/1_gcg_azure_ml.py` builds a config +The notebook at `doc/code/executor/gcg/1_gcg_azure_ml.py` builds a config locally, ships it to Azure ML as a job input, and the AML job invokes `run.py` with the path to the deserialized JSON. diff --git a/pyrit/auxiliary_attacks/gcg/experiments/__init__.py b/pyrit/executor/promptgen/gcg/experiments/__init__.py similarity index 100% rename from pyrit/auxiliary_attacks/gcg/experiments/__init__.py rename to pyrit/executor/promptgen/gcg/experiments/__init__.py diff --git a/pyrit/auxiliary_attacks/gcg/experiments/log.py b/pyrit/executor/promptgen/gcg/experiments/log.py similarity index 100% rename from pyrit/auxiliary_attacks/gcg/experiments/log.py rename to pyrit/executor/promptgen/gcg/experiments/log.py diff --git a/pyrit/auxiliary_attacks/gcg/experiments/run.py b/pyrit/executor/promptgen/gcg/experiments/run.py similarity index 93% rename from pyrit/auxiliary_attacks/gcg/experiments/run.py rename to pyrit/executor/promptgen/gcg/experiments/run.py index 396f46131d..3bc76d53c2 100644 --- a/pyrit/auxiliary_attacks/gcg/experiments/run.py +++ b/pyrit/executor/promptgen/gcg/experiments/run.py @@ -8,7 +8,7 @@ ``to_json_file`` methods, ships them to Azure ML as job inputs, and the job's command line is:: - python -m pyrit.auxiliary_attacks.gcg.experiments.run \\ + python -m pyrit.executor.promptgen.gcg.experiments.run \\ --config inputs/config.json \\ --data inputs/data.json \\ --output-dir ${{outputs.results}} @@ -23,9 +23,9 @@ from dataclasses import replace from pathlib import Path -from pyrit.auxiliary_attacks.gcg.config import GCGConfig, GCGDataConfig, GCGOutputConfig -from pyrit.auxiliary_attacks.gcg.data import load_goals_and_targets -from pyrit.auxiliary_attacks.gcg.generator import GCGGenerator +from pyrit.executor.promptgen.gcg.config import GCGConfig, GCGDataConfig, GCGOutputConfig +from pyrit.executor.promptgen.gcg.data import load_goals_and_targets +from pyrit.executor.promptgen.gcg.generator import GCGGenerator from pyrit.setup.initialization import _load_environment_files diff --git a/pyrit/auxiliary_attacks/gcg/extension_protocols.py b/pyrit/executor/promptgen/gcg/extension_protocols.py similarity index 96% rename from pyrit/auxiliary_attacks/gcg/extension_protocols.py rename to pyrit/executor/promptgen/gcg/extension_protocols.py index 973fb22a2b..8385bc98ae 100644 --- a/pyrit/auxiliary_attacks/gcg/extension_protocols.py +++ b/pyrit/executor/promptgen/gcg/extension_protocols.py @@ -24,7 +24,7 @@ Tensor-typed signatures are kept lazy via ``from __future__ import annotations`` plus a ``TYPE_CHECKING`` import for ``torch`` so that -``pyrit.auxiliary_attacks.gcg.extension_protocols`` itself imports cleanly on +``pyrit.executor.promptgen.gcg.extension_protocols`` itself imports cleanly on installs that only have the base ``dev`` extra (no torch). At call time the implementations are still operating on real ``torch.Tensor`` objects — the forward references just keep the runtime import side-effect free. @@ -61,7 +61,7 @@ class SamplingStrategy(Protocol): References: ``GCGPromptManager.sample_control`` in - ``pyrit/auxiliary_attacks/gcg/attack/gcg/gcg_attack.py``. + ``pyrit/executor/promptgen/gcg/attack/gcg/gcg_attack.py``. """ def sample_candidates( @@ -135,9 +135,9 @@ class LossFunction(Protocol): References: ``AttackPrompt.target_loss`` and ``AttackPrompt.control_loss`` in - ``pyrit/auxiliary_attacks/gcg/attack/base/attack_manager.py``, plus + ``pyrit/executor/promptgen/gcg/attack/base/attack_manager.py``, plus the weighted-sum aggregation inside ``GCGMultiPromptAttack.step`` in - ``pyrit/auxiliary_attacks/gcg/attack/gcg/gcg_attack.py``. + ``pyrit/executor/promptgen/gcg/attack/gcg/gcg_attack.py``. """ def compute_loss( @@ -195,7 +195,7 @@ class CandidateFilter(Protocol): References: ``MultiPromptAttack.get_filtered_cands`` in - ``pyrit/auxiliary_attacks/gcg/attack/base/attack_manager.py``. + ``pyrit/executor/promptgen/gcg/attack/base/attack_manager.py``. """ def filter_candidates( @@ -251,7 +251,7 @@ class SuffixInitializer(Protocol): References: ``AttackPrompt.__init__`` in - ``pyrit/auxiliary_attacks/gcg/attack/base/attack_manager.py`` assigns + ``pyrit/executor/promptgen/gcg/attack/base/attack_manager.py`` assigns ``self.control = control_init``. The same ``control_init`` parameter is threaded through the ``PromptManager``, ``MultiPromptAttack``, ``ProgressiveMultiPromptAttack``, ``IndividualPromptAttack``, and diff --git a/pyrit/auxiliary_attacks/gcg/generator.py b/pyrit/executor/promptgen/gcg/generator.py similarity index 98% rename from pyrit/auxiliary_attacks/gcg/generator.py rename to pyrit/executor/promptgen/gcg/generator.py index 12ef46040c..f4456d2da1 100644 --- a/pyrit/auxiliary_attacks/gcg/generator.py +++ b/pyrit/executor/promptgen/gcg/generator.py @@ -45,25 +45,25 @@ import torch.multiprocessing as mp from pydantic import Field -import pyrit.auxiliary_attacks.gcg.attack.gcg.gcg_attack as attack_lib -from pyrit.auxiliary_attacks.gcg.attack.base.attack_manager import ( +import pyrit.executor.promptgen.gcg.attack.gcg.gcg_attack as attack_lib +from pyrit.common.utils import combine_dict +from pyrit.executor.promptgen.core.prompt_generator_strategy import ( + PromptGeneratorStrategy, + PromptGeneratorStrategyContext, + PromptGeneratorStrategyResult, +) +from pyrit.executor.promptgen.gcg.attack.base.attack_manager import ( IndividualPromptAttack, ProgressiveMultiPromptAttack, get_workers, ) -from pyrit.auxiliary_attacks.gcg.config import ( +from pyrit.executor.promptgen.gcg.config import ( GCGAlgorithmConfig, GCGModelConfig, GCGOutputConfig, GCGStrategyConfig, ) -from pyrit.auxiliary_attacks.gcg.experiments.log import log_gpu_memory, log_train_goals -from pyrit.common.utils import combine_dict -from pyrit.executor.promptgen.core.prompt_generator_strategy import ( - PromptGeneratorStrategy, - PromptGeneratorStrategyContext, - PromptGeneratorStrategyResult, -) +from pyrit.executor.promptgen.gcg.experiments.log import log_gpu_memory, log_train_goals from pyrit.models import ComponentIdentifier, Identifiable logger = logging.getLogger(__name__) diff --git a/pyrit/auxiliary_attacks/gcg/src/Dockerfile b/pyrit/executor/promptgen/gcg/src/Dockerfile similarity index 100% rename from pyrit/auxiliary_attacks/gcg/src/Dockerfile rename to pyrit/executor/promptgen/gcg/src/Dockerfile diff --git a/tests/end_to_end/auxiliary_attacks/__init__.py b/tests/end_to_end/auxiliary_attacks/__init__.py deleted file mode 100644 index acd459977f..0000000000 --- a/tests/end_to_end/auxiliary_attacks/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ -# Copyright (c) Microsoft Corporation. -# Licensed under the MIT license. - -"""End-to-end tests for PyRIT auxiliary attacks (GCG).""" diff --git a/tests/end_to_end/auxiliary_attacks/test_gcg_aml_e2e.py b/tests/end_to_end/executor/promptgen/gcg/test_gcg_aml_e2e.py similarity index 96% rename from tests/end_to_end/auxiliary_attacks/test_gcg_aml_e2e.py rename to tests/end_to_end/executor/promptgen/gcg/test_gcg_aml_e2e.py index bf761f9ee8..83d5078d80 100644 --- a/tests/end_to_end/auxiliary_attacks/test_gcg_aml_e2e.py +++ b/tests/end_to_end/executor/promptgen/gcg/test_gcg_aml_e2e.py @@ -3,7 +3,7 @@ """End-to-end test for the GCG Azure ML pipeline. -Executes `doc/code/auxiliary_attacks/1_gcg_azure_ml.py` directly as a Python +Executes `doc/code/executor/gcg/1_gcg_azure_ml.py` directly as a Python script (the jupytext percent-format `# %%` markers are plain comments, so the file is valid Python). After the notebook submits the AML job, this test polls until the job reaches a terminal state and asserts success. @@ -55,7 +55,7 @@ "AZURE_ML_WORKSPACE_NAME", "HUGGINGFACE_TOKEN", ) -_NOTEBOOK_PATH = Path(HOME_PATH) / "doc" / "code" / "auxiliary_attacks" / "1_gcg_azure_ml.py" +_NOTEBOOK_PATH = Path(HOME_PATH) / "doc" / "code" / "executor" / "gcg" / "1_gcg_azure_ml.py" _DEFAULT_MAX_WAIT_SECONDS = 5400 # 90 minutes _POLL_INTERVAL_SECONDS = 30 _TERMINAL_STATES = {"Completed", "Failed", "Canceled", "CancelRequested"} diff --git a/tests/integration/auxiliary_attacks/test_gcg_attack_wiring_integration.py b/tests/integration/executor/promptgen/gcg/test_gcg_attack_wiring_integration.py similarity index 97% rename from tests/integration/auxiliary_attacks/test_gcg_attack_wiring_integration.py rename to tests/integration/executor/promptgen/gcg/test_gcg_attack_wiring_integration.py index 1a759d9b0a..9490ac9d56 100644 --- a/tests/integration/auxiliary_attacks/test_gcg_attack_wiring_integration.py +++ b/tests/integration/executor/promptgen/gcg/test_gcg_attack_wiring_integration.py @@ -28,22 +28,22 @@ import pytest attack_manager_mod = pytest.importorskip( - "pyrit.auxiliary_attacks.gcg.attack.base.attack_manager", + "pyrit.executor.promptgen.gcg.attack.base.attack_manager", reason="GCG optional dependencies (torch, mlflow, etc.) not installed", ) torch = pytest.importorskip("torch", reason="torch not installed") gcg_attack_mod = pytest.importorskip( - "pyrit.auxiliary_attacks.gcg.attack.gcg.gcg_attack", + "pyrit.executor.promptgen.gcg.attack.gcg.gcg_attack", reason="GCG optional dependencies not installed", ) generator_mod = pytest.importorskip( - "pyrit.auxiliary_attacks.gcg.generator", + "pyrit.executor.promptgen.gcg.generator", reason="GCG optional dependencies (torch, transformers, etc.) not installed", ) -from pyrit.auxiliary_attacks.gcg.config import ( # noqa: E402 +from pyrit.executor.promptgen.gcg.config import ( # noqa: E402 GCGAlgorithmConfig, GCGModelConfig, GCGOutputConfig, diff --git a/tests/integration/auxiliary_attacks/test_gcg_integration.py b/tests/integration/executor/promptgen/gcg/test_gcg_integration.py similarity index 98% rename from tests/integration/auxiliary_attacks/test_gcg_integration.py rename to tests/integration/executor/promptgen/gcg/test_gcg_integration.py index f2944bf480..a5ca3cdf78 100644 --- a/tests/integration/auxiliary_attacks/test_gcg_integration.py +++ b/tests/integration/executor/promptgen/gcg/test_gcg_integration.py @@ -25,13 +25,13 @@ from transformers import AutoTokenizer, GPT2LMHeadModel # noqa: E402 -from pyrit.auxiliary_attacks.gcg.attack.base.attack_manager import ( # noqa: E402 +from pyrit.executor.promptgen.gcg.attack.base.attack_manager import ( # noqa: E402 get_embedding_layer, get_embedding_matrix, get_embeddings, get_nonascii_toks, ) -from pyrit.auxiliary_attacks.gcg.attack.gcg.gcg_attack import ( # noqa: E402 +from pyrit.executor.promptgen.gcg.attack.gcg.gcg_attack import ( # noqa: E402 GCGAttackPrompt, GCGPromptManager, token_gradients, diff --git a/tests/integration/auxiliary_attacks/test_notebooks_auxiliary.py b/tests/integration/executor/promptgen/gcg/test_notebooks_gcg.py similarity index 90% rename from tests/integration/auxiliary_attacks/test_notebooks_auxiliary.py rename to tests/integration/executor/promptgen/gcg/test_notebooks_gcg.py index f1755912c4..e59098b802 100644 --- a/tests/integration/auxiliary_attacks/test_notebooks_auxiliary.py +++ b/tests/integration/executor/promptgen/gcg/test_notebooks_gcg.py @@ -10,7 +10,7 @@ from pyrit.common import path -nb_directory_path = pathlib.Path(path.DOCS_CODE_PATH, "auxiliary_attacks").resolve() +nb_directory_path = pathlib.Path(path.DOCS_CODE_PATH, "executor", "gcg").resolve() skipped_files = [ "1_gcg_azure_ml.ipynb", # missing required env variables diff --git a/tests/unit/auxiliary_attacks/__init__.py b/tests/unit/auxiliary_attacks/__init__.py deleted file mode 100644 index 9a0454564d..0000000000 --- a/tests/unit/auxiliary_attacks/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -# Copyright (c) Microsoft Corporation. -# Licensed under the MIT license. diff --git a/tests/unit/auxiliary_attacks/gcg/__init__.py b/tests/unit/auxiliary_attacks/gcg/__init__.py deleted file mode 100644 index 9a0454564d..0000000000 --- a/tests/unit/auxiliary_attacks/gcg/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -# Copyright (c) Microsoft Corporation. -# Licensed under the MIT license. diff --git a/tests/unit/auxiliary_attacks/test_experimental_warning.py b/tests/unit/auxiliary_attacks/test_experimental_warning.py deleted file mode 100644 index 693e5bf50b..0000000000 --- a/tests/unit/auxiliary_attacks/test_experimental_warning.py +++ /dev/null @@ -1,34 +0,0 @@ -# Copyright (c) Microsoft Corporation. -# Licensed under the MIT license. - -import importlib -import warnings - -import pyrit.auxiliary_attacks -from pyrit.exceptions import ExperimentalWarning - - -def test_importing_auxiliary_attacks_emits_experimental_warning() -> None: - with warnings.catch_warnings(record=True) as caught: - warnings.simplefilter("always") - importlib.reload(pyrit.auxiliary_attacks) - - experimental = [w for w in caught if issubclass(w.category, ExperimentalWarning)] - assert len(experimental) == 1 - message = str(experimental[0].message) - assert "pyrit.auxiliary_attacks is experimental" in message - assert "ExperimentalWarning" in message - - -def test_experimental_warning_is_a_future_warning_subclass() -> None: - assert issubclass(ExperimentalWarning, FutureWarning) - - -def test_experimental_warning_can_be_silenced() -> None: - with warnings.catch_warnings(record=True) as caught: - warnings.simplefilter("always") - warnings.filterwarnings("ignore", category=ExperimentalWarning) - importlib.reload(pyrit.auxiliary_attacks) - - experimental = [w for w in caught if issubclass(w.category, ExperimentalWarning)] - assert experimental == [] diff --git a/tests/unit/auxiliary_attacks/gcg/test_attack_manager_helpers.py b/tests/unit/executor/promptgen/gcg/test_attack_manager_helpers.py similarity index 98% rename from tests/unit/auxiliary_attacks/gcg/test_attack_manager_helpers.py rename to tests/unit/executor/promptgen/gcg/test_attack_manager_helpers.py index cf45cb4bdf..f1e2bdfaaa 100644 --- a/tests/unit/auxiliary_attacks/gcg/test_attack_manager_helpers.py +++ b/tests/unit/executor/promptgen/gcg/test_attack_manager_helpers.py @@ -8,7 +8,7 @@ import pytest attack_manager_mod = pytest.importorskip( - "pyrit.auxiliary_attacks.gcg.attack.base.attack_manager", + "pyrit.executor.promptgen.gcg.attack.base.attack_manager", reason="GCG optional dependencies (torch, mlflow, etc.) not installed", ) NpEncoder = attack_manager_mod.NpEncoder diff --git a/tests/unit/auxiliary_attacks/gcg/test_config.py b/tests/unit/executor/promptgen/gcg/test_config.py similarity index 99% rename from tests/unit/auxiliary_attacks/gcg/test_config.py rename to tests/unit/executor/promptgen/gcg/test_config.py index 922b0ffedd..09665f9733 100644 --- a/tests/unit/auxiliary_attacks/gcg/test_config.py +++ b/tests/unit/executor/promptgen/gcg/test_config.py @@ -13,7 +13,7 @@ import pytest -from pyrit.auxiliary_attacks.gcg.config import ( +from pyrit.executor.promptgen.gcg.config import ( GCGAlgorithmConfig, GCGConfig, GCGDataConfig, diff --git a/tests/unit/auxiliary_attacks/gcg/test_data_and_config.py b/tests/unit/executor/promptgen/gcg/test_data_and_config.py similarity index 93% rename from tests/unit/auxiliary_attacks/gcg/test_data_and_config.py rename to tests/unit/executor/promptgen/gcg/test_data_and_config.py index 793cf468ef..d41ffef2db 100644 --- a/tests/unit/auxiliary_attacks/gcg/test_data_and_config.py +++ b/tests/unit/executor/promptgen/gcg/test_data_and_config.py @@ -8,29 +8,29 @@ import pytest -from pyrit.auxiliary_attacks.gcg.config import GCGConfig, GCGDataConfig, GCGModelConfig, GCGOutputConfig +from pyrit.executor.promptgen.gcg.config import GCGConfig, GCGDataConfig, GCGModelConfig, GCGOutputConfig attack_manager_mod = pytest.importorskip( - "pyrit.auxiliary_attacks.gcg.attack.base.attack_manager", + "pyrit.executor.promptgen.gcg.attack.base.attack_manager", reason="GCG optional dependencies (torch, accelerate, etc.) not installed", ) get_goals_and_targets = attack_manager_mod.get_goals_and_targets data_mod = pytest.importorskip( - "pyrit.auxiliary_attacks.gcg.data", + "pyrit.executor.promptgen.gcg.data", reason="GCG data module requires torch (transitive via attack_manager)", ) load_goals_and_targets = data_mod.load_goals_and_targets run_mod = pytest.importorskip( - "pyrit.auxiliary_attacks.gcg.experiments.run", + "pyrit.executor.promptgen.gcg.experiments.run", reason="GCG run module not available", ) _main_async = run_mod._main_async _resolve_output = run_mod._resolve_output generator_mod = pytest.importorskip( - "pyrit.auxiliary_attacks.gcg.generator", + "pyrit.executor.promptgen.gcg.generator", reason="GCG generator module not available", ) GCGGenerator = generator_mod.GCGGenerator @@ -153,7 +153,7 @@ def test_override_falls_back_to_default_basename(self, tmp_path: Path) -> None: class TestMainAsyncCli: """Tests for ``run.py``'s ``--config`` + ``--data`` CLI wrapper around GCGGenerator.execute_async.""" - @patch("pyrit.auxiliary_attacks.gcg.experiments.run._load_environment_files") + @patch("pyrit.executor.promptgen.gcg.experiments.run._load_environment_files") async def test_raises_when_no_token_anywhere(self, mock_load_env: MagicMock, tmp_path: Path) -> None: config = GCGConfig(models=[GCGModelConfig(name="org/model")]) config_path = tmp_path / "config.json" @@ -166,8 +166,8 @@ async def test_raises_when_no_token_anywhere(self, mock_load_env: MagicMock, tmp with pytest.raises(ValueError, match="No HuggingFace token available"): await _main_async(str(config_path), str(data_path)) - @patch("pyrit.auxiliary_attacks.gcg.experiments.run._load_environment_files") - @patch("pyrit.auxiliary_attacks.gcg.experiments.run.load_goals_and_targets") + @patch("pyrit.executor.promptgen.gcg.experiments.run._load_environment_files") + @patch("pyrit.executor.promptgen.gcg.experiments.run.load_goals_and_targets") async def test_passes_loaded_goals_to_generator_and_uses_env_token( self, mock_loader: MagicMock, diff --git a/tests/unit/auxiliary_attacks/gcg/test_default_implementations.py b/tests/unit/executor/promptgen/gcg/test_default_implementations.py similarity index 94% rename from tests/unit/auxiliary_attacks/gcg/test_default_implementations.py rename to tests/unit/executor/promptgen/gcg/test_default_implementations.py index 8b89745052..1c79cdcd55 100644 --- a/tests/unit/auxiliary_attacks/gcg/test_default_implementations.py +++ b/tests/unit/executor/promptgen/gcg/test_default_implementations.py @@ -1,7 +1,7 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT license. -"""Tests for ``pyrit.auxiliary_attacks.gcg.default_implementations``. +"""Tests for ``pyrit.executor.promptgen.gcg.default_implementations``. These tests verify byte-identical parity between the four default implementations and the legacy GCG attack code paths they reproduce: @@ -15,7 +15,7 @@ inside ``AttackPrompt.__init__`` Mocking patterns follow the conventions established in -``tests/unit/auxiliary_attacks/gcg/test_gcg_core.py`` (``object.__new__`` +``tests/unit/executor/promptgen/gcg/test_gcg_core.py`` (``object.__new__`` to skip the real ``__init__``, ``MagicMock`` tokenizers). """ @@ -26,23 +26,23 @@ torch = pytest.importorskip("torch", reason="GCG default implementations require torch") attack_manager_mod = pytest.importorskip( - "pyrit.auxiliary_attacks.gcg.attack.base.attack_manager", + "pyrit.executor.promptgen.gcg.attack.base.attack_manager", reason="GCG optional dependencies (torch, mlflow, etc.) not installed", ) gcg_attack_mod = pytest.importorskip( - "pyrit.auxiliary_attacks.gcg.attack.gcg.gcg_attack", + "pyrit.executor.promptgen.gcg.attack.gcg.gcg_attack", reason="GCG optional dependencies not installed", ) -import pyrit.auxiliary_attacks.gcg as gcg_pkg # noqa: E402 -from pyrit.auxiliary_attacks.gcg import ( # noqa: E402 +import pyrit.executor.promptgen.gcg as gcg_pkg # noqa: E402 +from pyrit.executor.promptgen.gcg import ( # noqa: E402 CrossEntropyLoss, LengthPreservingFilter, LiteralStringInit, StandardGCGSampling, ) -from pyrit.auxiliary_attacks.gcg import default_implementations as defaults_module # noqa: E402 -from pyrit.auxiliary_attacks.gcg.config import GCGAlgorithmConfig # noqa: E402 +from pyrit.executor.promptgen.gcg import default_implementations as defaults_module # noqa: E402 +from pyrit.executor.promptgen.gcg.config import GCGAlgorithmConfig # noqa: E402 AttackPrompt = attack_manager_mod.AttackPrompt MultiPromptAttack = attack_manager_mod.MultiPromptAttack @@ -65,8 +65,8 @@ def test_default_is_reexported_with_identity(self, name: str) -> None: package_attr = getattr(gcg_pkg, name) module_attr = getattr(defaults_module, name) assert package_attr is module_attr, ( - f"{name} re-exported from pyrit.auxiliary_attacks.gcg must be the same " - f"object as pyrit.auxiliary_attacks.gcg.default_implementations.{name}" + f"{name} re-exported from pyrit.executor.promptgen.gcg must be the same " + f"object as pyrit.executor.promptgen.gcg.default_implementations.{name}" ) @pytest.mark.parametrize("name", DEFAULT_NAMES) @@ -95,7 +95,7 @@ def _make_legacy_prompt_manager( def test_sample_candidates_matches_legacy_with_ascii_only(self) -> None: """Legacy reference: ``GCGPromptManager.sample_control(grad, batch_size, topk=top_k, temp=1.0, allow_non_ascii=False)`` in - ``pyrit/auxiliary_attacks/gcg/attack/gcg/gcg_attack.py``. + ``pyrit/executor/promptgen/gcg/attack/gcg/gcg_attack.py``. """ n_control_tokens = 5 vocab_size = 50 @@ -200,7 +200,7 @@ def test_compute_loss_matches_legacy_weighted_sum(self) -> None: ``target_weight * AttackPrompt.target_loss(logits, ids).mean(dim=-1)`` ``+ control_weight * AttackPrompt.control_loss(logits, ids).mean(dim=-1)``, per ``GCGMultiPromptAttack.step`` in - ``pyrit/auxiliary_attacks/gcg/attack/gcg/gcg_attack.py``. + ``pyrit/executor/promptgen/gcg/attack/gcg/gcg_attack.py``. """ batch_size = 4 seq_len = 10 @@ -359,7 +359,7 @@ def test_filter_candidates_matches_legacy_filtered(self) -> None: """Legacy reference: ``MultiPromptAttack.get_filtered_cands(0, control_cand, filter_cand=True, curr_control=...)`` in - ``pyrit/auxiliary_attacks/gcg/attack/base/attack_manager.py``. + ``pyrit/executor/promptgen/gcg/attack/base/attack_manager.py``. With the helper tokenizer: - Row 0 ``[3, 0, 1]`` -> decode ``"xxx"`` (len 3); retok len 3 == diff --git a/tests/unit/auxiliary_attacks/gcg/test_extension_protocols.py b/tests/unit/executor/promptgen/gcg/test_extension_protocols.py similarity index 92% rename from tests/unit/auxiliary_attacks/gcg/test_extension_protocols.py rename to tests/unit/executor/promptgen/gcg/test_extension_protocols.py index 404ef4b541..1aa3c0335d 100644 --- a/tests/unit/auxiliary_attacks/gcg/test_extension_protocols.py +++ b/tests/unit/executor/promptgen/gcg/test_extension_protocols.py @@ -1,7 +1,7 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT license. -"""Tests for :mod:`pyrit.auxiliary_attacks.gcg.extension_protocols`. +"""Tests for :mod:`pyrit.executor.promptgen.gcg.extension_protocols`. These are typing-surface tests: they verify the four ``Protocol``s are exposed, are ``runtime_checkable``, and accept a minimal in-test concrete @@ -19,14 +19,14 @@ # file is skipped on installs that only have the base ``dev`` extra. torch = pytest.importorskip("torch", reason="GCG extension protocols reference torch.Tensor") -import pyrit.auxiliary_attacks.gcg as gcg_pkg # noqa: E402 -from pyrit.auxiliary_attacks.gcg import ( # noqa: E402 +import pyrit.executor.promptgen.gcg as gcg_pkg # noqa: E402 +from pyrit.executor.promptgen.gcg import ( # noqa: E402 CandidateFilter, LossFunction, SamplingStrategy, SuffixInitializer, ) -from pyrit.auxiliary_attacks.gcg import extension_protocols as protocols_module # noqa: E402 +from pyrit.executor.promptgen.gcg import extension_protocols as protocols_module # noqa: E402 PROTOCOL_NAMES = ( "CandidateFilter", @@ -45,8 +45,8 @@ def test_protocols_are_reexported_from_package_with_identity() -> None: package_attr = getattr(gcg_pkg, name) module_attr = getattr(protocols_module, name) assert package_attr is module_attr, ( - f"{name} re-exported from pyrit.auxiliary_attacks.gcg must be the same " - f"object as pyrit.auxiliary_attacks.gcg.extension_protocols.{name}" + f"{name} re-exported from pyrit.executor.promptgen.gcg must be the same " + f"object as pyrit.executor.promptgen.gcg.extension_protocols.{name}" ) diff --git a/tests/unit/auxiliary_attacks/gcg/test_gcg_core.py b/tests/unit/executor/promptgen/gcg/test_gcg_core.py similarity index 99% rename from tests/unit/auxiliary_attacks/gcg/test_gcg_core.py rename to tests/unit/executor/promptgen/gcg/test_gcg_core.py index a90563ed85..3c25054844 100644 --- a/tests/unit/auxiliary_attacks/gcg/test_gcg_core.py +++ b/tests/unit/executor/promptgen/gcg/test_gcg_core.py @@ -7,7 +7,7 @@ import pytest attack_manager_mod = pytest.importorskip( - "pyrit.auxiliary_attacks.gcg.attack.base.attack_manager", + "pyrit.executor.promptgen.gcg.attack.base.attack_manager", reason="GCG optional dependencies (torch, mlflow, etc.) not installed", ) torch = pytest.importorskip("torch", reason="torch not installed") @@ -23,7 +23,7 @@ get_embeddings = attack_manager_mod.get_embeddings gcg_attack_mod = pytest.importorskip( - "pyrit.auxiliary_attacks.gcg.attack.gcg.gcg_attack", + "pyrit.executor.promptgen.gcg.attack.gcg.gcg_attack", reason="GCG optional dependencies not installed", ) GCGMultiPromptAttack = gcg_attack_mod.GCGMultiPromptAttack diff --git a/tests/unit/auxiliary_attacks/gcg/test_generator.py b/tests/unit/executor/promptgen/gcg/test_generator.py similarity index 98% rename from tests/unit/auxiliary_attacks/gcg/test_generator.py rename to tests/unit/executor/promptgen/gcg/test_generator.py index 956dcc4953..f35e90db9a 100644 --- a/tests/unit/auxiliary_attacks/gcg/test_generator.py +++ b/tests/unit/executor/promptgen/gcg/test_generator.py @@ -12,7 +12,7 @@ import pytest -from pyrit.auxiliary_attacks.gcg.config import ( +from pyrit.executor.promptgen.gcg.config import ( GCGAlgorithmConfig, GCGModelConfig, GCGOutputConfig, @@ -23,7 +23,7 @@ from pathlib import Path generator_mod = pytest.importorskip( - "pyrit.auxiliary_attacks.gcg.generator", + "pyrit.executor.promptgen.gcg.generator", reason="GCG optional dependencies (torch, transformers, etc.) not installed", ) GCGGenerator = generator_mod.GCGGenerator @@ -357,7 +357,7 @@ def filter_candidates( patch.object(gen, "_create_attack", return_value=fake_attack) as mock_create_attack, patch.object(gen, "_build_logfile_path", return_value=str(tmp_path / "result.json")), patch.object(gen, "_read_result", return_value=GCGResult(final_suffix="x")), - patch("pyrit.auxiliary_attacks.gcg.generator.asyncio.to_thread", new=AsyncMock(return_value=None)), + patch("pyrit.executor.promptgen.gcg.generator.asyncio.to_thread", new=AsyncMock(return_value=None)), ): await gen._perform_async(context=context) diff --git a/tests/unit/auxiliary_attacks/gcg/test_get_goals_and_targets.py b/tests/unit/executor/promptgen/gcg/test_get_goals_and_targets.py similarity index 98% rename from tests/unit/auxiliary_attacks/gcg/test_get_goals_and_targets.py rename to tests/unit/executor/promptgen/gcg/test_get_goals_and_targets.py index 9a6c0c93da..08c78df1a4 100644 --- a/tests/unit/auxiliary_attacks/gcg/test_get_goals_and_targets.py +++ b/tests/unit/executor/promptgen/gcg/test_get_goals_and_targets.py @@ -8,7 +8,7 @@ import pytest attack_manager_mod = pytest.importorskip( - "pyrit.auxiliary_attacks.gcg.attack.base.attack_manager", + "pyrit.executor.promptgen.gcg.attack.base.attack_manager", reason="GCG optional dependencies (torch, mlflow, etc.) not installed", ) get_goals_and_targets = attack_manager_mod.get_goals_and_targets diff --git a/tests/unit/auxiliary_attacks/gcg/test_log.py b/tests/unit/executor/promptgen/gcg/test_log.py similarity index 92% rename from tests/unit/auxiliary_attacks/gcg/test_log.py rename to tests/unit/executor/promptgen/gcg/test_log.py index da50956ed6..400fced0d7 100644 --- a/tests/unit/auxiliary_attacks/gcg/test_log.py +++ b/tests/unit/executor/promptgen/gcg/test_log.py @@ -7,7 +7,7 @@ import pytest log_mod = pytest.importorskip( - "pyrit.auxiliary_attacks.gcg.experiments.log", + "pyrit.executor.promptgen.gcg.experiments.log", reason="GCG optional dependencies not installed", ) get_gpu_memory = log_mod.get_gpu_memory @@ -90,28 +90,28 @@ class TestGpuMemoryLogging: uses stdlib imports, so these tests run in any CI environment. """ - @patch("pyrit.auxiliary_attacks.gcg.experiments.log.sp") + @patch("pyrit.executor.promptgen.gcg.experiments.log.sp") def test_get_gpu_memory_parses_nvidia_smi(self, mock_sp: MagicMock) -> None: """Should parse nvidia-smi output into a dict of GPU -> free memory.""" mock_sp.check_output.return_value = b"memory.free [MiB]\n8000 MiB\n16000 MiB\n" result = get_gpu_memory() assert result == {"gpu1_free_memory": 8000, "gpu2_free_memory": 16000} - @patch("pyrit.auxiliary_attacks.gcg.experiments.log.sp") + @patch("pyrit.executor.promptgen.gcg.experiments.log.sp") def test_get_gpu_memory_single_gpu(self, mock_sp: MagicMock) -> None: """Should handle single GPU output.""" mock_sp.check_output.return_value = b"memory.free [MiB]\n24000 MiB\n" result = get_gpu_memory() assert result == {"gpu1_free_memory": 24000} - @patch("pyrit.auxiliary_attacks.gcg.experiments.log.sp") + @patch("pyrit.executor.promptgen.gcg.experiments.log.sp") def test_log_gpu_memory_logs_via_logging(self, mock_sp: MagicMock) -> None: """Should log GPU memory info without error on the success path.""" mock_sp.check_output.return_value = b"memory.free [MiB]\n8000 MiB\n16000 MiB\n" # Should not raise log_gpu_memory(step=5) - @patch("pyrit.auxiliary_attacks.gcg.experiments.log.sp") + @patch("pyrit.executor.promptgen.gcg.experiments.log.sp") def test_log_gpu_memory_swallows_nvidia_smi_failure(self, mock_sp: MagicMock) -> None: """Should swallow exceptions when nvidia-smi is not available. @@ -123,7 +123,7 @@ def test_log_gpu_memory_swallows_nvidia_smi_failure(self, mock_sp: MagicMock) -> # Must not raise log_gpu_memory(step=5) - @patch("pyrit.auxiliary_attacks.gcg.experiments.log.sp") + @patch("pyrit.executor.promptgen.gcg.experiments.log.sp") def test_get_gpu_memory_handles_nvidia_smi_failure(self, mock_sp: MagicMock) -> None: """`get_gpu_memory` itself should propagate the exception (only `log_gpu_memory` is expected to swallow it).""" diff --git a/tests/unit/auxiliary_attacks/gcg/test_multi_prompt_attack.py b/tests/unit/executor/promptgen/gcg/test_multi_prompt_attack.py similarity index 98% rename from tests/unit/auxiliary_attacks/gcg/test_multi_prompt_attack.py rename to tests/unit/executor/promptgen/gcg/test_multi_prompt_attack.py index 0eeef9e859..6100643399 100644 --- a/tests/unit/auxiliary_attacks/gcg/test_multi_prompt_attack.py +++ b/tests/unit/executor/promptgen/gcg/test_multi_prompt_attack.py @@ -6,7 +6,7 @@ import pytest attack_manager_mod = pytest.importorskip( - "pyrit.auxiliary_attacks.gcg.attack.base.attack_manager", + "pyrit.executor.promptgen.gcg.attack.base.attack_manager", reason="GCG optional dependencies (torch, mlflow, etc.) not installed", ) IndividualPromptAttack = attack_manager_mod.IndividualPromptAttack diff --git a/tests/unit/auxiliary_attacks/gcg/test_public_api.py b/tests/unit/executor/promptgen/gcg/test_public_api.py similarity index 86% rename from tests/unit/auxiliary_attacks/gcg/test_public_api.py rename to tests/unit/executor/promptgen/gcg/test_public_api.py index a0b114eb73..9f637fffa4 100644 --- a/tests/unit/auxiliary_attacks/gcg/test_public_api.py +++ b/tests/unit/executor/promptgen/gcg/test_public_api.py @@ -1,7 +1,7 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT license. -"""Tests for the top-level :mod:`pyrit.auxiliary_attacks.gcg` public API surface.""" +"""Tests for the top-level :mod:`pyrit.executor.promptgen.gcg` public API surface.""" import pytest @@ -10,8 +10,8 @@ # Skip the whole file on installs that only have the base `dev` extra. pytest.importorskip("torch", reason="GCG public API exposes torch-dependent symbols") -import pyrit.auxiliary_attacks.gcg as gcg_pkg -from pyrit.auxiliary_attacks.gcg import ( +import pyrit.executor.promptgen.gcg as gcg_pkg +from pyrit.executor.promptgen.gcg import ( GCG, GCGAlgorithmConfig, GCGConfig, @@ -49,7 +49,7 @@ def test_public_api_symbols_are_exported() -> None: def test_public_api_symbols_are_importable_from_package() -> None: # Smoke-test that the imports at module top resolved to real objects so the - # short import path (e.g. ``from pyrit.auxiliary_attacks.gcg import GCG``) + # short import path (e.g. ``from pyrit.executor.promptgen.gcg import GCG``) # stays stable. symbols = ( GCG, From 2e7b38c3f34fd862f737ee8f91aeaa871f073e45 Mon Sep 17 00:00:00 2001 From: Copilot <223556219+Copilot@users.noreply.github.com> Date: Tue, 14 Jul 2026 20:40:57 -0700 Subject: [PATCH 2/4] Link GCG source from the AML notebook via absolute GitHub URLs The AML notebook linked into the package source tree with relative paths (../../../../pyrit/...). Those 404 once the notebook is rendered on the docs website, which does not ship the pyrit/ package tree alongside the docs. Switch both links (the Dockerfile and the run.py -m entry point) to absolute https://github.com/microsoft/PyRIT/blob/main/... URLs, matching how the rest of doc/ links to source files. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 4c446ad2-7050-4b03-bd88-dfe83ec69434 --- doc/code/executor/gcg/1_gcg_azure_ml.ipynb | 4 ++-- doc/code/executor/gcg/1_gcg_azure_ml.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/code/executor/gcg/1_gcg_azure_ml.ipynb b/doc/code/executor/gcg/1_gcg_azure_ml.ipynb index f2eeeda8a3..bb0f021272 100644 --- a/doc/code/executor/gcg/1_gcg_azure_ml.ipynb +++ b/doc/code/executor/gcg/1_gcg_azure_ml.ipynb @@ -130,7 +130,7 @@ "metadata": {}, "source": [ "To install the dependencies needed to run GCG, we create an AML environment from a\n", - "[Dockerfile](../../../../pyrit/executor/promptgen/gcg/src/Dockerfile). The Dockerfile uses\n", + "[Dockerfile](https://github.com/microsoft/PyRIT/blob/main/pyrit/executor/promptgen/gcg/src/Dockerfile). The Dockerfile uses\n", "an NVIDIA CUDA base image with Python 3.11 and installs PyRIT with the `gcg` extra." ] }, @@ -187,7 +187,7 @@ "metadata": {}, "source": [ "Finally, we configure the command to run the GCG algorithm. The entry point is\n", - "[`pyrit.executor.promptgen.gcg.experiments.run`](../../../../pyrit/executor/promptgen/gcg/experiments/run.py),\n", + "[`pyrit.executor.promptgen.gcg.experiments.run`](https://github.com/microsoft/PyRIT/blob/main/pyrit/executor/promptgen/gcg/experiments/run.py),\n", "invoked as a module so the uploaded code snapshot takes priority over the\n", "Docker-installed package (Python's `-m` flag puts the cwd at the front of `sys.path`).\n", "\n", diff --git a/doc/code/executor/gcg/1_gcg_azure_ml.py b/doc/code/executor/gcg/1_gcg_azure_ml.py index a229f1ce14..985ba6399c 100644 --- a/doc/code/executor/gcg/1_gcg_azure_ml.py +++ b/doc/code/executor/gcg/1_gcg_azure_ml.py @@ -65,7 +65,7 @@ # %% [markdown] # To install the dependencies needed to run GCG, we create an AML environment from a -# [Dockerfile](../../../../pyrit/executor/promptgen/gcg/src/Dockerfile). The Dockerfile uses +# [Dockerfile](https://github.com/microsoft/PyRIT/blob/main/pyrit/executor/promptgen/gcg/src/Dockerfile). The Dockerfile uses # an NVIDIA CUDA base image with Python 3.11 and installs PyRIT with the `gcg` extra. # %% @@ -94,7 +94,7 @@ # %% [markdown] # Finally, we configure the command to run the GCG algorithm. The entry point is -# [`pyrit.executor.promptgen.gcg.experiments.run`](../../../../pyrit/executor/promptgen/gcg/experiments/run.py), +# [`pyrit.executor.promptgen.gcg.experiments.run`](https://github.com/microsoft/PyRIT/blob/main/pyrit/executor/promptgen/gcg/experiments/run.py), # invoked as a module so the uploaded code snapshot takes priority over the # Docker-installed package (Python's `-m` flag puts the cwd at the front of `sys.path`). # From e182e9f61846dc5e8da6d1ffc1d46bf3b9f153a8 Mon Sep 17 00:00:00 2001 From: Copilot <223556219+Copilot@users.noreply.github.com> Date: Tue, 14 Jul 2026 20:47:17 -0700 Subject: [PATCH 3/4] Keep GCG experimental: warn on import from its new location The GCG package moved out of pyrit/auxiliary_attacks/ (which emitted an ExperimentalWarning on import) into pyrit/executor/promptgen/gcg/. GCG is still experimental -- its API should be free to change without a deprecation cycle -- so re-emit the ExperimentalWarning from the new package __init__. The warning fires only on an explicit `import pyrit.executor.promptgen.gcg` (or `from ... import`), not on `import pyrit.executor.promptgen`, so the torch-free base import stays silent. Restores the warning unit test at the new path. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 4c446ad2-7050-4b03-bd88-dfe83ec69434 --- pyrit/executor/promptgen/gcg/__init__.py | 19 +++++++++++ .../gcg/test_experimental_warning.py | 34 +++++++++++++++++++ 2 files changed, 53 insertions(+) create mode 100644 tests/unit/executor/promptgen/gcg/test_experimental_warning.py diff --git a/pyrit/executor/promptgen/gcg/__init__.py b/pyrit/executor/promptgen/gcg/__init__.py index 3becd552ab..05f3743af5 100644 --- a/pyrit/executor/promptgen/gcg/__init__.py +++ b/pyrit/executor/promptgen/gcg/__init__.py @@ -23,10 +23,20 @@ goals=["how do I ..."], targets=["Sure, here is ..."], ) + +This subpackage is **experimental**: APIs may change in any release without a +deprecation cycle. Pin pyrit to a specific version if you depend on it. To +silence the warning emitted on import:: + + import warnings + from pyrit.exceptions import ExperimentalWarning + warnings.filterwarnings("ignore", category=ExperimentalWarning) """ +import warnings from typing import TYPE_CHECKING, Any +from pyrit.exceptions import ExperimentalWarning from pyrit.executor.promptgen.gcg.config import ( GCGAlgorithmConfig, GCGConfig, @@ -36,6 +46,15 @@ GCGStrategyConfig, ) +warnings.warn( + "pyrit.executor.promptgen.gcg is experimental: APIs may change in any release " + "without a deprecation cycle. Pin pyrit to a specific version if you depend " + "on this module. To silence: " + "warnings.filterwarnings('ignore', category=pyrit.exceptions.ExperimentalWarning).", + ExperimentalWarning, + stacklevel=2, +) + # Torch-dependent symbols are exposed lazily via PEP 562 __getattr__ so that # `from pyrit.executor.promptgen.gcg import GCGConfig` works on installs that # only have the base `dev` extra (no torch). Touching any of these names from diff --git a/tests/unit/executor/promptgen/gcg/test_experimental_warning.py b/tests/unit/executor/promptgen/gcg/test_experimental_warning.py new file mode 100644 index 0000000000..29cff9b408 --- /dev/null +++ b/tests/unit/executor/promptgen/gcg/test_experimental_warning.py @@ -0,0 +1,34 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import importlib +import warnings + +import pyrit.executor.promptgen.gcg +from pyrit.exceptions import ExperimentalWarning + + +def test_importing_gcg_emits_experimental_warning() -> None: + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + importlib.reload(pyrit.executor.promptgen.gcg) + + experimental = [w for w in caught if issubclass(w.category, ExperimentalWarning)] + assert len(experimental) == 1 + message = str(experimental[0].message) + assert "pyrit.executor.promptgen.gcg is experimental" in message + assert "ExperimentalWarning" in message + + +def test_experimental_warning_is_a_future_warning_subclass() -> None: + assert issubclass(ExperimentalWarning, FutureWarning) + + +def test_experimental_warning_can_be_silenced() -> None: + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + warnings.filterwarnings("ignore", category=ExperimentalWarning) + importlib.reload(pyrit.executor.promptgen.gcg) + + experimental = [w for w in caught if issubclass(w.category, ExperimentalWarning)] + assert experimental == [] From 96ac3d6dec73a690e2dc8593b51992dd8d106ec5 Mon Sep 17 00:00:00 2001 From: Copilot <223556219+Copilot@users.noreply.github.com> Date: Tue, 14 Jul 2026 20:58:44 -0700 Subject: [PATCH 4/4] Drop lingering 'auxiliary attack' wording from GCG docstring GCG is now a PromptGeneratorStrategy in the standard framework, not an auxiliary attack; the docstring's descriptive phrasing was the last textual vestige of the removed pyrit/auxiliary_attacks subtree. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 4c446ad2-7050-4b03-bd88-dfe83ec69434 --- pyrit/executor/promptgen/gcg/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyrit/executor/promptgen/gcg/__init__.py b/pyrit/executor/promptgen/gcg/__init__.py index 05f3743af5..6e58eb52d1 100644 --- a/pyrit/executor/promptgen/gcg/__init__.py +++ b/pyrit/executor/promptgen/gcg/__init__.py @@ -1,7 +1,7 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT license. -"""Public API for the Greedy Coordinate Gradient (GCG) auxiliary attack. +"""Public API for the Greedy Coordinate Gradient (GCG) adversarial-suffix generator. The primary entry point is ``GCG`` (alias for ``GCGGenerator``), a ``pyrit.executor.promptgen.core.PromptGeneratorStrategy`` that produces