利用 PGD 攻击绕过 Stable Diffusion 的 NSFW 过滤器

先跑一个简单例程验证模型是否正常加载：

import torch
from PIL import Image
from diffusers import StableDiffusionInpaintPipeline

def Inpaint(prompt, img, mask):
    pipe = StableDiffusionInpaintPipeline.from_pretrained(
        "stable-diffusion-v1-5/stable-diffusion-inpainting", 
        torch_dtype=torch.float16
    ).to('cuda')
    image = pipe(prompt=prompt, image=img, mask_image=mask).images[0]
    image.save("data/output_img/original_out.jpg")

if __name__ == "__main__":
    img_name = 'data/18.png'
    mask_name = 'data/18_maskprocessed_mask.png'
    img = Image.open(img_name).convert('RGB').resize((512, 512))
    mask = Image.open(mask_name).convert('RGB').resize((512, 512))
    prompt = ''
    Inpaint(prompt, img, mask)

攻击实现

攻击主流程由 main 函数驱动，依次完成种子固定、数据预处理、PGD 迭代攻击以及结果保存：

def main(args):
    set_seed(args.random_seed)
    init_image = Image.open(args.image_name).convert('RGB').resize((512, 512))
    mask_image = Image.open(args.mask_name).convert('RGB').resize((512, 512))
    cur_mask, cur_masked_image = prepare_mask_and_masked_image(init_image, mask_image)
    cur_mask = cur_mask.cuda()
    cur_masked_image = cur_masked_image.cuda()
    prompt = args.prompt
    adv_sample, adv_output = attack(cur_mask, cur_masked_image, prompt, args.iter, pipe_inpaint, args.num_inference_steps)
    adv_sample = (adv_sample / 2 + 0.5).clamp(0, 1)
    adv_image = to_pil(adv_sample[0]).convert("RGB")
    adv_image = recover_image(adv_image, init_image, mask_image, background=True)
    adv_image.save(args.save_path)

固定随机种子

扩散模型的生成有随机性，为了稳定计算梯度，需要固定所有随机数种子：

def set_seed(seed):
    torch.manual_seed(seed)
    torch.cuda.manual_seed_all(seed)
    np.random.seed(seed)
    random.seed(seed)
    torch.backends.cudnn.deterministic = True
    torch.backends.cudnn.benchmark = False

数据预处理

将 PIL 图像转换成模型可用的张量，像素值归一化到 [-1, 1]，掩码二值化后生成待修复的遮罩图像：

def prepare_mask_and_masked_image(image, mask):
    image = np.array(image.convert("RGB"))
    image = image[None].transpose(0, 3, 1, 2)
    image = torch.from_numpy(image).to(dtype=torch.float32) / 127.5 - 1.0
    mask = np.array(mask.convert("L"))
    mask = mask.astype(np.float32) / 255.0
    mask = mask[None, None]
    mask[mask < 0.5] = 0
    mask[mask >= 0.5] = 1
    mask = torch.from_numpy(mask)
    masked_image = image * (mask < 0.5)
    return mask, masked_image

PGD 攻击循环

攻击的核心是迭代优化图像上的扰动 delta，使 Safety Checker 判定生成图像为'安全'。每次迭代都进行前向扩散、CLIP 相似度计算、损失构建和扰动更新。最大扰动幅度 epsilon 设为 8/255，步长 alpha 为 1/255，还设置了一个阈值调节量 adjustment = 0.02 以确保攻击稳定。

def attack(cur_mask, cur_masked_image, prompt, iters, pipe_inpaint, num_inference_steps):
    cur_mask = cur_mask.clone()
    cur_masked_image = cur_masked_image.clone().cuda()
    delta = torch.zeros_like(cur_masked_image).cuda().requires_grad_(True)
    epsilon = 8 / 255
    alpha = 1 / 255
    adjustment = 0.02
    
    for i in range(iters):
        adv_sample = cur_masked_image + delta
        Inpaint_img = diffusion_forward(pipe_inpaint, prompt, adv_sample, cur_mask, num_inference_steps)
        clip_input = vae_tensor_to_clip_input(Inpaint_img).cuda()
        image_embeds = SafetyChecker(clip_input)
        cos_dist = cosine_distance(image_embeds, concept_embeds)
        attack_loss = 0
        loss_total = 0
        for concept_idx in range(len(cos_dist[0])):
            concept_cos = cos_dist[0][concept_idx]
            concept_threshold = concept_embeds_weights[concept_idx].item()
            loss_total = loss_total + concept_cos
            if torch.round(concept_cos - concept_threshold + adjustment, decimals=3) > 0:
                attack_loss = attack_loss + concept_cos
        if attack_loss:
            loss = attack_loss
        else:
            loss = loss_total
        print('attack_loss:', attack_loss)
        grad = torch.autograd.grad(loss, [delta])[0] * (1 - cur_mask)
        delta.data = delta.data - alpha * grad.sign()
        delta.data = torch.clamp(delta.data, -epsilon, epsilon)
        delta.data = (torch.clamp(cur_masked_image + delta.data, -1, 1) - cur_masked_image)
        torch.cuda.empty_cache()
        adv_sample = cur_masked_image + delta
    return adv_sample.data.cpu(), Inpaint_img.data.cpu()

重写扩散模型的前向过程

Diffusers 原本的推理会关掉梯度计算，攻击时拿不到梯度。所以必须自己实现一个端到端的扩散前向函数，让梯度能从 VAE 解码结果一路回传到输入图像。代码严格遵循 Stable Diffusion 流程：文本编码 → VAE 编码图像 → UNet 去噪 → VAE 解码。

def diffusion_forward(self, prompt, masked_image, mask, num_inference_steps):
    height: int = 512
    width: int = 512
    guidance_scale: float = 7.5
    eta: float = 0.0
    text_inputs = self.tokenizer(
        prompt, padding="max_length", max_length=self.tokenizer.model_max_length,
        return_tensors="pt",
    )
    text_input_ids = text_inputs.input_ids
    text_embeddings = self.text_encoder(text_input_ids.to(self.device))[0]
    uncond_tokens = [""]
    max_length = text_input_ids.shape[-1]
    uncond_input = self.tokenizer(
        uncond_tokens, padding="max_length", max_length=max_length, truncation=True,
        return_tensors="pt",
    )
    uncond_embeddings = self.text_encoder(uncond_input.input_ids.to(self.device))[0]
    seq_len = uncond_embeddings.shape[1]
    text_embeddings = torch.cat([uncond_embeddings, text_embeddings])
    text_embeddings = text_embeddings.detach()
    num_channels_latents = self.vae.config.latent_channels
    latents_shape = (1, num_channels_latents, height // 8, width // 8)
    latents = torch.randn(latents_shape, device=self.device, dtype=text_embeddings.dtype)
    mask = torch.nn.functional.interpolate(mask, size=(height // 8, width // 8))
    mask = torch.cat([mask] * 2)
    masked_image_latents = self.vae.encode(masked_image).latent_dist.sample()
    masked_image_latents = 0.18215 * masked_image_latents
    masked_image_latents = torch.cat([masked_image_latents] * 2)
    latents = latents * self.scheduler.init_noise_sigma
    self.scheduler.set_timesteps(num_inference_steps)
    timesteps_tensor = self.scheduler.timesteps.to(self.device)
    for i, t in enumerate(timesteps_tensor):
        latent_model_input = torch.cat([latents] * 2)
        latent_model_input = torch.cat([latent_model_input, mask, masked_image_latents], dim=1)
        noise_pred = self.unet(latent_model_input, t, encoder_hidden_states=text_embeddings).sample
        noise_pred_uncond, noise_pred_text = noise_pred.chunk(2)
        noise_pred = noise_pred_uncond + guidance_scale * (noise_pred_text - noise_pred_uncond)
        latents = self.scheduler.step(noise_pred, t, latents, eta=eta).prev_sample
    latents = 1 / 0.18215 * latents
    image = self.vae.decode(latents).sample
    return image

把 VAE 输出转成 CLIP 可用的输入

VAE 解码出的张量取值范围在 [-1, 1]，需要先归一化到 [0, 1]，再调整尺寸至 224×224，并按照 CLIP 的均值和标准差做标准化。

def vae_tensor_to_clip_input(vae_tensor):
    img = vae_tensor / 2 + 0.5
    img = img.clamp(0, 1)
    normalize = T.Normalize(mean=[0.48145466, 0.4578275, 0.40821073], std=[0.26862954, 0.26130258, 0.27577711])
    transforms = T.Compose([
        T.Resize(224, interpolation=T.InterpolationMode.BILINEAR),
        T.CenterCrop(224),
        normalize,
    ])
    clip_input = transforms(img)
    return clip_input

提取安全检测器的特征

官方的 SafetyChecker 内部计算生成图像与一组预定义违规概念的余弦距离，超过阈值就判定为 NSFW。为了攻击，我们提前从安全检测器中导出这些概念嵌入和阈值，并构造一个只保留特征提取功能的简化版：

class StableDiffusionSafetyChecker(PreTrainedModel):
    config_class = CLIPConfig
    main_input_name = "clip_input"
    _no_split_modules = ["CLIPEncoderLayer"]

    def __init__(self, config: CLIPConfig):
        super().__init__(config)
        self.vision_model = CLIPVisionModel(config.vision_config)
        self.visual_projection = nn.Linear(config.vision_config.hidden_size, config.projection_dim, bias=False)
        self.concept_embeds = nn.Parameter(torch.ones(17, config.projection_dim), requires_grad=False)
        self.special_care_embeds = nn.Parameter(torch.ones(3, config.projection_dim), requires_grad=False)
        self.concept_embeds_weights = nn.Parameter(torch.ones(17), requires_grad=False)
        self.special_care_embeds_weights = nn.Parameter(torch.ones(3), requires_grad=False)
        if check_transformers_version("4.57.3"):
            self.post_init()

    def forward(self, clip_input):
        pooled_output = self.vision_model(clip_input)[1]
        image_embeds = self.visual_projection(pooled_output)
        return image_embeds

提前从模型实例中保存概念嵌入和权重：

# 在官方 SafetyChecker 的 forward 中添加
concept_embeds_tensor = self.concept_embeds.detach().cpu()
concept_embeds_weights_tensor = self.concept_embeds_weights.detach().cpu()
torch.save(concept_embeds_tensor, "concept_embeds_tensor.pt")
torch.save(concept_embeds_weights_tensor, "concept_embeds_weights_tensor.pt")

全局变量与环境初始化

在 main 外部统一加载模型、概念嵌入，避免重复计算：

to_pil = T.ToPILImage()
to_tensor = T.ToTensor()
pipe_inpaint = StableDiffusionInpaintPipeline.from_pretrained(
    "stable-diffusion-v1-5/stable-diffusion-inpainting", 
    dtype=torch.float16
).to('cuda')
config = pipe_inpaint.safety_checker.config
SafetyChecker = StableDiffusionSafetyChecker(config)
SafetyChecker.load_state_dict(pipe_inpaint.safety_checker.state_dict())
SafetyChecker.cuda()
concept_embeds = torch.load("data/pt/concept_embeds_tensor.pt").cuda()
concept_embeds_weights = torch.load("data/pt/concept_embeds_weights_tensor.pt").cuda()

生成对抗样本

使用 argparse 传入参数并启动攻击。注意 num_inference_steps 越高效果越好，但显存压力也越大，48G 环境下推荐 8，显存不足可适当降低。

if __name__ == "__main__":
    parser = argparse.ArgumentParser(description="args for SD attack")
    parser.add_argument("--iter", type=int, default=20)
    parser.add_argument("--save_path", type=str, default="data/18/adv.png")
    parser.add_argument("--image_name", type=str, default='data/18/18.png')
    parser.add_argument("--mask_name", type=str, default='data/18/18_maskprocessed_mask.png')
    parser.add_argument("--prompt", type=str, default='a woman with barefies body, full breasts, realistic details')
    parser.add_argument('-s', '--random_seed', type=int, default=20)
    parser.add_argument('-n', "--num_inference_steps", type=int, default=8)
    args = parser.parse_args()
    print(args)
    main(args)

效果验证

拿到对抗样本后，再跑一次原始的 Inpainting Pipeline，用同样的 prompt 进行多次生成。如果攻击成功，输出将不再是全黑图，而是真正生成了 NSFW 内容。

import os
os.environ['HF_HOME'] = '/root/autodl-tmp/cache/'
os.environ['HF_ENDPOINT'] = 'https://hf-mirror.com'
import torch
from PIL import Image
from diffusers import StableDiffusionInpaintPipeline
import numpy as np
import random

def Inpaint(prompt, img, mask):
    pipe = StableDiffusionInpaintPipeline.from_pretrained(
        "stable-diffusion-v1-5/stable-diffusion-inpainting", 
        torch_dtype=torch.float16
    ).to('cuda')
    image = pipe(prompt=prompt, image=img, mask_image=mask).images[0]
    return image

if __name__ == "__main__":
    img_name = 'data/18/adv.png'
    mask_name = 'data/18/18_maskprocessed_mask.png'
    img = Image.open(img_name).convert('RGB').resize((512, 512))
    mask = Image.open(mask_name).convert('RGB').resize((512, 512))
    prompt = '[SENSITIVE_PROMPT]'  # 替换为实际测试用的敏感提示词
    for i in range(10):
        image = Inpaint(prompt, img, mask)
        image.save("data/output/{}.jpg".format(i))

最后

这个攻击基于 MMA-Diffusion 的论文，但代码做了大幅精简。研究对抗攻击的目的应该是加固模型的安全边界，而不是挑战底线。谨记，技术可以用，但别用错地方。