From 33d13ac6e63a08cbecc9b987950427bb4dc23a2d Mon Sep 17 00:00:00 2001 From: Raja Sekhar Rao Dheekonda Date: Tue, 20 Oct 2026 12:52:02 -0700 Subject: [PATCH 1/4] feat(ai-red-teaming): wire AutoDAN attack (v1.18.9) Adds autodan_attack to the attack_runner registry - AutoDAN (Liu et al. ICLR 2024), a closed-box genetic-algorithm readable jailbreak that runs against hosted dn/ models like any other jailbreak. The other new white-box attacks (GCG family, embedding, activation) are intentionally SDK-only: they need dreadnode[whitebox] + a local open-weight surrogate for gradients, which the hosted inference sandbox does not provide. Paired with dreadnode-tiger SDK PR #2857. --- capabilities/ai-red-teaming/capability.yaml | 2 +- .../ai-red-teaming/scripts/attack_runner.py | 15 +++++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/capabilities/ai-red-teaming/capability.yaml b/capabilities/ai-red-teaming/capability.yaml index fa53f6d..dbb2734 100644 --- a/capabilities/ai-red-teaming/capability.yaml +++ b/capabilities/ai-red-teaming/capability.yaml @@ -1,6 +1,6 @@ schema: 1 name: ai-red-teaming -version: "1.18.8" +version: "1.18.9" description: > Probe the security and safety of AI applications, agents, and foundation models. Orchestrates adversarial attack workflows to discover vulnerabilities in LLMs, diff --git a/capabilities/ai-red-teaming/scripts/attack_runner.py b/capabilities/ai-red-teaming/scripts/attack_runner.py index 4ccb928..931ceed 100644 --- a/capabilities/ai-red-teaming/scripts/attack_runner.py +++ b/capabilities/ai-red-teaming/scripts/attack_runner.py @@ -491,6 +491,21 @@ def _auto_execute_workflow(filename: str, timeout: int = 3600) -> str: "top_k_strategies": 5, }, }, + # AutoDAN (Liu et al. ICLR 2024): genetic-algorithm readable jailbreak. Closed-box + # (needs only text output + a judge), so it runs against hosted dn/ models like any + # other jailbreak. The other white-box attacks (GCG family, embedding, activation) + # are SDK-only: they require dreadnode[whitebox] + a LOCAL open-weight surrogate for + # gradients, which the hosted inference sandbox does not provide. + "autodan_attack": { + "module": "autodan", + "function": "autodan_attack", + "has_attacker": True, + "default_iterations": 50, + "extra_defaults": { + "early_stopping_score": 0.9, + "beam_width": 8, + }, + }, "renellm_attack": { "module": "renellm", "function": "renellm_attack", From 2092a8e9fb834f697179fd8a5816e3067b2b7b3d Mon Sep 17 00:00:00 2001 From: Raja Sekhar Rao Dheekonda Date: Tue, 20 Oct 2026 15:21:27 -0700 Subject: [PATCH 2/4] feat(ai-red-teaming): generalize attack registry via SDK auto-discovery (v1.19.0) Replaces hand-maintained attack wiring with auto-discovery: _discover_attack_defs() introspects dreadnode.airt for standard-signature (goal/target/evaluator_model) *_attack functions and derives module/function/has_attacker/iterations/defaults from each signature. New SDK attacks now flow into the capability with no manual entry. The manual table is kept as tuned overrides (merged on top). White-box gradient attacks are gated out (_WHITEBOX_LOCAL_ONLY) since they need a local surrogate the hosted sandbox can't provide. Verified: 41 attacks auto-discovered from the installed SDK, white-box correctly excluded, manual overrides preserved. --- capabilities/ai-red-teaming/capability.yaml | 2 +- .../ai-red-teaming/scripts/attack_runner.py | 77 ++++++++++++++++++- 2 files changed, 77 insertions(+), 2 deletions(-) diff --git a/capabilities/ai-red-teaming/capability.yaml b/capabilities/ai-red-teaming/capability.yaml index dbb2734..36dc93d 100644 --- a/capabilities/ai-red-teaming/capability.yaml +++ b/capabilities/ai-red-teaming/capability.yaml @@ -1,6 +1,6 @@ schema: 1 name: ai-red-teaming -version: "1.18.9" +version: "1.19.0" description: > Probe the security and safety of AI applications, agents, and foundation models. Orchestrates adversarial attack workflows to discover vulnerabilities in LLMs, diff --git a/capabilities/ai-red-teaming/scripts/attack_runner.py b/capabilities/ai-red-teaming/scripts/attack_runner.py index 931ceed..2086a42 100644 --- a/capabilities/ai-red-teaming/scripts/attack_runner.py +++ b/capabilities/ai-red-teaming/scripts/attack_runner.py @@ -390,7 +390,7 @@ def _auto_execute_workflow(filename: str, timeout: int = 3600) -> str: "databricks llama": "databricks/databricks-meta-llama-3-1-70b-instruct", } -_ATTACK_DEFS: dict[str, dict] = { +_MANUAL_ATTACK_DEFS: dict[str, dict] = { "tap_attack": { "module": "tap", "function": "tap_attack", @@ -943,6 +943,81 @@ def _auto_execute_workflow(filename: str, timeout: int = 3600) -> str: }, } +# White-box gradient attacks are SDK-only: they require a LOCAL open-weight +# surrogate for gradients, which the hosted inference sandbox cannot provide, so +# they are excluded from auto-discovery (they would crash at runtime here). +_WHITEBOX_LOCAL_ONLY = frozenset( + { + "gcg_attack", + "faster_gcg_attack", + "i_gcg_attack", + "attngcg_attack", + "mask_gcg_attack", + "gbda_attack", + "pez_attack", + "cold_attack", + "soft_gcg_attack", + "arca_attack", + "amplegcg_attack", + "refusal_ablation_attack", + "soft_prompt_attack", + "latentbreak_attack", + } +) + + +def _discover_attack_defs() -> dict[str, dict]: + """Auto-discover text-jailbreak attacks from the installed SDK so new attacks + flow into the capability without hand-wiring. Only functions with the standard + ``(goal, target, evaluator_model, ...)`` signature are included (skips + evasion/extraction/inversion/image attacks, which take a different target), and + white-box gradient attacks are excluded (no local surrogate in the sandbox). + Defaults are read from each function's signature; the manual table above + overrides these for attacks with specially tuned parameters.""" + import inspect + + try: + import dreadnode.airt as _airt + except Exception: + return {} + + discovered: dict[str, dict] = {} + for _name in dir(_airt): + if not _name.endswith("_attack") or _name in _WHITEBOX_LOCAL_ONLY: + continue + _fn = getattr(_airt, _name, None) + if not callable(_fn): + continue + try: + _params = inspect.signature(_fn).parameters + except (TypeError, ValueError): + continue + if not ({"goal", "target", "evaluator_model"} <= set(_params)): + continue + + def _default(key: str, fallback: object, _p: object = _params) -> object: + value = _p[key].default if key in _p else inspect.Parameter.empty + return fallback if value is inspect.Parameter.empty else value + + extra: dict[str, object] = {} + early_stop = _default("early_stopping_score", None) + if early_stop is not None: + extra["early_stopping_score"] = early_stop + + discovered[_name] = { + "module": _fn.__module__.rsplit(".", 1)[-1], + "function": _name, + "has_attacker": "attacker_model" in _params, + "default_iterations": _default("n_iterations", 20), + "extra_defaults": extra, + } + return discovered + + +# Auto-discovered base (all standard-signature SDK attacks), with the hand-tuned +# manual table taking precedence where an attack needs specific defaults. +_ATTACK_DEFS: dict[str, dict] = {**_discover_attack_defs(), **_MANUAL_ATTACK_DEFS} + # Short aliases -> canonical attack name ATTACK_ALIASES: dict[str, str] = {} for _canon, _def in _ATTACK_DEFS.items(): From fa3c250d26b57e81197b49a47122a2e4913f0168 Mon Sep 17 00:00:00 2001 From: Raja Sekhar Rao Dheekonda Date: Tue, 20 Oct 2026 16:12:48 -0700 Subject: [PATCH 3/4] feat(ai-red-teaming): unified white-box support - discover + local-surrogate skill (v1.20.0) White-box attacks are no longer gated out of discovery; they are included and flagged requires_surrogate so SDK and TUI share one surface. The user supplies a local open-weight surrogate as attacker_model (the SDK already errors clearly if a dn/* model is passed - no gradients over the proxy). Adds the whitebox-local-surrogate skill: how to run them, local-runtime/GPU + dreadnode[whitebox] requirements, trust_remote_code + resource guardrails, and when to prefer closed-box instead. --- capabilities/ai-red-teaming/capability.yaml | 2 +- .../ai-red-teaming/scripts/attack_runner.py | 24 +++++--- .../skills/whitebox-local-surrogate/SKILL.md | 55 +++++++++++++++++++ 3 files changed, 71 insertions(+), 10 deletions(-) create mode 100644 capabilities/ai-red-teaming/skills/whitebox-local-surrogate/SKILL.md diff --git a/capabilities/ai-red-teaming/capability.yaml b/capabilities/ai-red-teaming/capability.yaml index 36dc93d..fa77e75 100644 --- a/capabilities/ai-red-teaming/capability.yaml +++ b/capabilities/ai-red-teaming/capability.yaml @@ -1,6 +1,6 @@ schema: 1 name: ai-red-teaming -version: "1.19.0" +version: "1.20.0" description: > Probe the security and safety of AI applications, agents, and foundation models. Orchestrates adversarial attack workflows to discover vulnerabilities in LLMs, diff --git a/capabilities/ai-red-teaming/scripts/attack_runner.py b/capabilities/ai-red-teaming/scripts/attack_runner.py index 2086a42..7d7b1a7 100644 --- a/capabilities/ai-red-teaming/scripts/attack_runner.py +++ b/capabilities/ai-red-teaming/scripts/attack_runner.py @@ -943,10 +943,13 @@ def _auto_execute_workflow(filename: str, timeout: int = 3600) -> str: }, } -# White-box gradient attacks are SDK-only: they require a LOCAL open-weight -# surrogate for gradients, which the hosted inference sandbox cannot provide, so -# they are excluded from auto-discovery (they would crash at runtime here). -_WHITEBOX_LOCAL_ONLY = frozenset( +# White-box gradient attacks optimize against a LOCAL open-weight surrogate: the +# user must pass that model path/id as `attacker_model` (see the +# whitebox-local-surrogate skill). They are discoverable (unified SDK + TUI +# support), but flagged `requires_surrogate` so the agent knows to supply a local +# model and run in local runtime. In the hosted sandbox with a dn/* attacker, the +# SDK raises a clear error (no gradients over the proxy) rather than crashing. +_WHITEBOX_REQUIRES_SURROGATE = frozenset( { "gcg_attack", "faster_gcg_attack", @@ -970,10 +973,12 @@ def _discover_attack_defs() -> dict[str, dict]: """Auto-discover text-jailbreak attacks from the installed SDK so new attacks flow into the capability without hand-wiring. Only functions with the standard ``(goal, target, evaluator_model, ...)`` signature are included (skips - evasion/extraction/inversion/image attacks, which take a different target), and - white-box gradient attacks are excluded (no local surrogate in the sandbox). - Defaults are read from each function's signature; the manual table above - overrides these for attacks with specially tuned parameters.""" + evasion/extraction/inversion/image attacks, which take a different target). + White-box gradient attacks ARE included for unified SDK + TUI support, flagged + ``requires_surrogate`` so the agent supplies a local model path as + ``attacker_model`` (see the whitebox-local-surrogate skill). Defaults are read + from each function's signature; the manual table above overrides these for + attacks with specially tuned parameters.""" import inspect try: @@ -983,7 +988,7 @@ def _discover_attack_defs() -> dict[str, dict]: discovered: dict[str, dict] = {} for _name in dir(_airt): - if not _name.endswith("_attack") or _name in _WHITEBOX_LOCAL_ONLY: + if not _name.endswith("_attack"): continue _fn = getattr(_airt, _name, None) if not callable(_fn): @@ -1010,6 +1015,7 @@ def _default(key: str, fallback: object, _p: object = _params) -> object: "has_attacker": "attacker_model" in _params, "default_iterations": _default("n_iterations", 20), "extra_defaults": extra, + "requires_surrogate": _name in _WHITEBOX_REQUIRES_SURROGATE, } return discovered diff --git a/capabilities/ai-red-teaming/skills/whitebox-local-surrogate/SKILL.md b/capabilities/ai-red-teaming/skills/whitebox-local-surrogate/SKILL.md new file mode 100644 index 0000000..bd02e8b --- /dev/null +++ b/capabilities/ai-red-teaming/skills/whitebox-local-surrogate/SKILL.md @@ -0,0 +1,55 @@ +--- +name: whitebox-local-surrogate +description: How to run white-box (gradient/embedding/activation) attacks, which need a local open-weight surrogate model the user supplies +allowed-tools: register_assessment generate_attack +--- + +# White-box attacks: local surrogate + +White-box attacks (the GCG family, embedding-space, generator, and activation +attacks) do not just read the target's text - they optimize against model +internals (gradients, logits, hidden states). That requires a **local open-weight +surrogate model**, which the user provides. The optimized prompt then transfers to +the closed-box target (`dn/*` or BYO API) as usual. + +Attacks that need a surrogate: `gcg`, `faster_gcg`, `i_gcg`, `attngcg`, `mask_gcg`, +`arca`, `gbda`, `pez`, `cold`, `soft_gcg`, `amplegcg`, `refusal_ablation`, +`soft_prompt`, `latentbreak`. (`autodan` is closed-box and needs no surrogate.) + +## How to run one + +Ask the user for the surrogate (a Hugging Face id or local path) and pass it as +`attacker_model`; keep the real model as `target_model`: + +``` +generate_attack( + attack="gcg", + goal="...", + target_model="dn/claude-opus-4-6", # closed-box transfer target + attacker_model="meta-llama/Llama-2-7b-chat-hf", # LOCAL open-weight surrogate + evaluator_model="dn/claude-opus-4-6", +) +``` + +For `refusal_ablation` / `soft_prompt` / `latentbreak` (local-only: no transferable +text), the surrogate IS the attacked model and it is judged on its own generation. + +## Requirements and guardrails + +- **Local runtime only.** These need the surrogate's weights and (for 7B+) a GPU. + The managed hosted sandbox has neither - run white-box attacks in a local or + self-hosted runtime. If a `dn/*` model is passed as `attacker_model`, the SDK + raises a clear error (no gradients over the inference proxy); switch to a local id. +- **Install the extra:** `pip install "dreadnode[whitebox]"` (torch + transformers). +- **Trust:** do not enable `trust_remote_code` for untrusted model repos (it runs + code on load). +- **Resources:** 7B-scale surrogates need a GPU with ~80 GB; small models run on CPU + for experimentation. `early_stopping_score` keeps cost bounded - easy targets exit + fast; only hard ones run the full `n_iterations`. +- Prefer an **aligned open-weight** surrogate (e.g. Llama-2-7b-chat) for meaningful + transfer; a tiny random model produces junk suffixes. + +## When NOT to use + +If the user only has a hosted model and no local weights/GPU, use closed-box attacks +instead (`tap`, `pair`, `crescendo`, `autodan`, ...). See the attack-selection-guide. From a313151b106b098fbeefbbad47e050593f86a353 Mon Sep 17 00:00:00 2001 From: Raja Sekhar Rao Dheekonda Date: Tue, 20 Oct 2026 17:30:01 -0700 Subject: [PATCH 4/4] feat(ai-red-teaming): flag adaptive_random_search as white-box (requires surrogate) --- capabilities/ai-red-teaming/scripts/attack_runner.py | 1 + 1 file changed, 1 insertion(+) diff --git a/capabilities/ai-red-teaming/scripts/attack_runner.py b/capabilities/ai-red-teaming/scripts/attack_runner.py index 7d7b1a7..99b3bf2 100644 --- a/capabilities/ai-red-teaming/scripts/attack_runner.py +++ b/capabilities/ai-red-teaming/scripts/attack_runner.py @@ -965,6 +965,7 @@ def _auto_execute_workflow(filename: str, timeout: int = 3600) -> str: "refusal_ablation_attack", "soft_prompt_attack", "latentbreak_attack", + "adaptive_random_search_attack", } )