[Qwen3.5] Project the refusal direction out of the residual stream at runtime
orcarouter/Qwen3.8-27B-Uncensored removes refusal by editing weights. The Swift release we serve is AutoRound W8A16, so re-editing weights means re-quantizing. Project one unit direction r out of every residual-stream writer output instead: x <- x - r (r . x), which equals the weight edit W' = W - r(r^T W) on every writer feeding that stream (Arditi et al. 2024). Sites: attention output for both the full-attention and GDN branches, the MLP output, and token embeddings, in Qwen3NextDecoderLayer.forward and Qwen3NextModel.embed_input_ids, which Qwen3_5DecoderLayer and Qwen3_5Model inherit but whose base constructors they skip, so both register there too. The MTP drafter reuses Qwen3_5DecoderLayer and calls its own embed_input_ids, so the draft path is covered at the same width, and _maybe_share_embeddings at PP=1 hands the draft the target embedding module. Sequence parallelism shards dim 0 only, so the last dim is hidden_size at every site. VLLM_ABLITERATION names a JSON file with key refusal_direction, a safetensors tensor, or a bare name resolved under /opt/qwen38/abliteration then /data/abliteration. Empty or 0 is off. register() always creates the buffer _abliteration_r, zeros when off, so one compiled graph serves ON and OFF and toggling the switch (declared in envs.py, hence in compile_factors) invalidates the compile cache. project_() is an in-place rank-1 update with no full-size temporary, and skips tensors whose width is not hidden_size. Co-authored-by: Hinata <hinata@idihore.id> Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
@@ -0,0 +1,151 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Optional runtime refusal-direction removal ("abliteration").
|
||||
|
||||
``VLLM_ABLITERATION`` (or the legacy ``QWEN38_ABLITERATION``) names a JSON file
|
||||
with a ``refusal_direction`` list, or a safetensors file with that tensor, of
|
||||
length ``hidden_size``. A bare name resolves to ``<name>.json`` in
|
||||
``/opt/qwen38/abliteration``, then in ``/data/abliteration``. When set, every
|
||||
write into the residual stream loses its component along the unit direction r:
|
||||
|
||||
x <- x - r (r . x)
|
||||
|
||||
This runs on the attention block outputs (full and linear attention), the MLP
|
||||
output and the text token embeddings. For one direction it equals the weight
|
||||
edit ``W' = W - r (r^T W)`` on ``self_attn.o_proj``,
|
||||
``linear_attn.out_proj``, ``mlp.down_proj`` and the rows of ``embed_tokens``
|
||||
(Arditi et al., 2024), so the served weights, quantized or not, stay untouched.
|
||||
|
||||
Unset, no projection kernel runs. The buffer always exists, holding zeros while
|
||||
the switch is off, so the attribute a compiled or captured graph refers to is
|
||||
always present, and the switch is part of the torch.compile cache key, so
|
||||
toggling it never reuses a stale graph.
|
||||
"""
|
||||
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
from functools import lru_cache
|
||||
|
||||
import torch
|
||||
|
||||
from vllm import envs
|
||||
from vllm.logger import init_logger
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
ENV = "VLLM_ABLITERATION"
|
||||
BUILTIN_DIRS = ("/opt/qwen38/abliteration", "/data/abliteration")
|
||||
DIRECTION = "_abliteration_r"
|
||||
ENABLED = "_abliteration_on"
|
||||
|
||||
_shape_warned = False
|
||||
|
||||
|
||||
def direction_path() -> str | None:
|
||||
"""The configured direction file, or None when the switch is off."""
|
||||
value = (envs.VLLM_ABLITERATION or "").strip()
|
||||
if value in ("", "0"):
|
||||
return None
|
||||
if os.sep in value or value.endswith((".json", ".safetensors")):
|
||||
return value
|
||||
for directory in BUILTIN_DIRS:
|
||||
path = os.path.join(directory, f"{value}.json")
|
||||
if os.path.exists(path):
|
||||
return path
|
||||
return os.path.join(BUILTIN_DIRS[0], f"{value}.json")
|
||||
|
||||
|
||||
@lru_cache(maxsize=4)
|
||||
def _read(path: str) -> torch.Tensor:
|
||||
"""The direction as a unit float32 CPU tensor, read once per path."""
|
||||
if path.endswith(".json"):
|
||||
with open(path) as f:
|
||||
values = json.load(f)["refusal_direction"]
|
||||
# A model built on the meta device would otherwise make this CPU tensor
|
||||
# a meta one, and a meta tensor cannot be inspected or copied.
|
||||
r = torch.tensor(values, dtype=torch.float32, device="cpu")
|
||||
else:
|
||||
from safetensors import safe_open
|
||||
|
||||
with safe_open(path, framework="pt", device="cpu") as f:
|
||||
r = f.get_tensor("refusal_direction")
|
||||
r = r.to(dtype=torch.float32, device="cpu")
|
||||
r = r.flatten()
|
||||
norm = float(r.norm())
|
||||
if not math.isfinite(norm) or norm == 0.0:
|
||||
raise ValueError(f"{ENV}: {path} holds a zero or non-finite direction")
|
||||
logger.info("%s: refusal direction (%d values) from %s", ENV, r.numel(), path)
|
||||
return r / norm
|
||||
|
||||
|
||||
def load_direction(hidden_size: int) -> torch.Tensor | None:
|
||||
"""The unit refusal direction on CPU, or None when the switch is off."""
|
||||
path = direction_path()
|
||||
if path is None:
|
||||
return None
|
||||
r = _read(path)
|
||||
if r.numel() != hidden_size:
|
||||
raise ValueError(
|
||||
f"{ENV}: {path} holds {r.numel()} values; the model's hidden size "
|
||||
f"is {hidden_size}"
|
||||
)
|
||||
return r
|
||||
|
||||
|
||||
def register(module: torch.nn.Module, hidden_size: int, dtype: torch.dtype) -> None:
|
||||
"""Attach the direction as a non-persistent buffer, from a model constructor.
|
||||
|
||||
Call it before CUDA-graph capture so the buffer lands on the model's device.
|
||||
It holds zeros when the switch is off. A buffer that lands on the meta
|
||||
device cannot hold a direction, which turns the projection off for that
|
||||
module.
|
||||
"""
|
||||
r = load_direction(hidden_size)
|
||||
buf = torch.zeros(hidden_size, dtype=dtype)
|
||||
on = r is not None
|
||||
if on and buf.device.type == "meta":
|
||||
logger.warning(
|
||||
"%s: %s is built on the meta device, which cannot hold a direction; "
|
||||
"projection is off for this module",
|
||||
ENV,
|
||||
type(module).__name__,
|
||||
)
|
||||
on = False
|
||||
elif on:
|
||||
buf.copy_(r)
|
||||
module.register_buffer(DIRECTION, buf, persistent=False)
|
||||
setattr(module, ENABLED, on)
|
||||
|
||||
|
||||
def project_(module: torch.nn.Module, x: torch.Tensor) -> torch.Tensor:
|
||||
"""Remove the component of x along the direction, in place. No-op when off."""
|
||||
if not getattr(module, ENABLED, False):
|
||||
return x
|
||||
r = getattr(module, DIRECTION)
|
||||
width = x.shape[-1]
|
||||
if width != r.numel():
|
||||
_warn_shape(type(module).__name__, width, r.numel())
|
||||
return x
|
||||
if r.dtype != x.dtype:
|
||||
# Activations do not always carry the model dtype, and the rank-1 update
|
||||
# needs one dtype throughout.
|
||||
r = r.to(x.dtype)
|
||||
if x.is_contiguous():
|
||||
flat = x.view(-1, width)
|
||||
flat.addr_(flat @ r, r, alpha=-1.0)
|
||||
else:
|
||||
x.sub_(torch.outer(x.reshape(-1, width) @ r, r).view(x.shape))
|
||||
return x
|
||||
|
||||
|
||||
def _warn_shape(name: str, width: int, size: int) -> None:
|
||||
global _shape_warned
|
||||
if not _shape_warned:
|
||||
_shape_warned = True
|
||||
logger.warning(
|
||||
"%s: skipped a %d-wide tensor from %s; the direction has %d values",
|
||||
ENV,
|
||||
width,
|
||||
name,
|
||||
size,
|
||||
)
|
||||
@@ -66,6 +66,7 @@ from vllm.multimodal.inputs import MultiModalFeatureSpec
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.tokenizers.registry import cached_tokenizer_from_config
|
||||
|
||||
from . import abliteration
|
||||
from .interfaces import (
|
||||
HasInnerState,
|
||||
IsHybrid,
|
||||
@@ -191,6 +192,7 @@ class Qwen3_5DecoderLayer(Qwen3NextDecoderLayer):
|
||||
self.post_attention_layernorm = Qwen3_5RMSNorm(
|
||||
config.hidden_size, eps=config.rms_norm_eps
|
||||
)
|
||||
abliteration.register(self, config.hidden_size, model_config.dtype)
|
||||
|
||||
self.layer_scale = getattr(config, "layer_scale", False)
|
||||
if self.layer_scale:
|
||||
@@ -254,6 +256,7 @@ class Qwen3_5Model(Qwen3NextModel):
|
||||
quant_config=self.quant_config,
|
||||
prefix=f"{prefix}.embed_tokens",
|
||||
)
|
||||
abliteration.register(self, config.hidden_size, vllm_config.model_config.dtype)
|
||||
|
||||
def get_layer(prefix: str):
|
||||
return Qwen3_5DecoderLayer(
|
||||
|
||||
@@ -37,6 +37,7 @@ from vllm.model_executor.models.utils import sequence_parallel_chunk
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.transformers_utils.repo_utils import try_get_local_file
|
||||
|
||||
from . import abliteration
|
||||
from .interfaces import (
|
||||
MultiModalEmbeddings,
|
||||
SupportsMultiModal,
|
||||
@@ -146,6 +147,7 @@ class Qwen3_5MultiTokenPredictor(nn.Module):
|
||||
self.vocab_size,
|
||||
config.hidden_size,
|
||||
)
|
||||
abliteration.register(self, config.hidden_size, model_config.dtype)
|
||||
|
||||
# Workaround: mtp.fc is stored as BF16 in NVFP4 checkpoints but is
|
||||
# missing from hf_quant_config.json exclude_modules. Force unquantized.
|
||||
@@ -217,7 +219,7 @@ class Qwen3_5MultiTokenPredictor(nn.Module):
|
||||
)
|
||||
|
||||
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
|
||||
return self.embed_tokens(input_ids)
|
||||
return abliteration.project_(self, self.embed_tokens(input_ids))
|
||||
|
||||
def forward(
|
||||
self,
|
||||
|
||||
@@ -61,6 +61,7 @@ from vllm.platforms import current_platform
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.v1.attention.backend import AttentionType
|
||||
|
||||
from . import abliteration
|
||||
from .interfaces import (
|
||||
EagleModelMixin,
|
||||
HasInnerState,
|
||||
@@ -537,6 +538,7 @@ class Qwen3NextDecoderLayer(nn.Module):
|
||||
self.post_attention_layernorm = Qwen3NextRMSNorm(
|
||||
config.hidden_size, eps=config.rms_norm_eps
|
||||
)
|
||||
abliteration.register(self, config.hidden_size, model_config.dtype)
|
||||
|
||||
self.layer_scale = getattr(config, "layer_scale", False)
|
||||
if self.layer_scale:
|
||||
@@ -584,6 +586,8 @@ class Qwen3NextDecoderLayer(nn.Module):
|
||||
else:
|
||||
raise ValueError("Invalid layer_type")
|
||||
|
||||
abliteration.project_(self, hidden_states)
|
||||
|
||||
if self.layer_scale:
|
||||
if len(hidden_states.shape) == 2:
|
||||
hidden_states = hidden_states * (
|
||||
@@ -612,6 +616,8 @@ class Qwen3NextDecoderLayer(nn.Module):
|
||||
else:
|
||||
hidden_states = self.mlp(hidden_states)
|
||||
|
||||
abliteration.project_(self, hidden_states)
|
||||
|
||||
if self.layer_scale:
|
||||
if len(hidden_states.shape) == 2:
|
||||
hidden_states = hidden_states * (
|
||||
@@ -661,6 +667,7 @@ class Qwen3NextModel(nn.Module, EagleModelMixin):
|
||||
self.vocab_size,
|
||||
config.hidden_size,
|
||||
)
|
||||
abliteration.register(self, config.hidden_size, vllm_config.model_config.dtype)
|
||||
|
||||
def get_layer(prefix: str):
|
||||
return Qwen3NextDecoderLayer(
|
||||
@@ -689,7 +696,7 @@ class Qwen3NextModel(nn.Module, EagleModelMixin):
|
||||
self.aux_hidden_state_layers: tuple[int, ...] = ()
|
||||
|
||||
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
|
||||
return self.embed_tokens(input_ids)
|
||||
return abliteration.project_(self, self.embed_tokens(input_ids))
|
||||
|
||||
@property
|
||||
def use_sequence_parallel(self) -> bool:
|
||||
|
||||
Reference in New Issue
Block a user