[Qwen3.5] Project the refusal direction out of the residual stream at runtime

orcarouter/Qwen3.8-27B-Uncensored removes refusal by editing weights. The
Swift release we serve is AutoRound W8A16, so re-editing weights means
re-quantizing. Project one unit direction r out of every residual-stream
writer output instead: x <- x - r (r . x), which equals the weight edit
W' = W - r(r^T W) on every writer feeding that stream (Arditi et al. 2024).

Sites: attention output for both the full-attention and GDN branches, the
MLP output, and token embeddings, in Qwen3NextDecoderLayer.forward and
Qwen3NextModel.embed_input_ids, which Qwen3_5DecoderLayer and Qwen3_5Model
inherit but whose base constructors they skip, so both register there too.
The MTP drafter reuses Qwen3_5DecoderLayer and calls its own
embed_input_ids, so the draft path is covered at the same width, and
_maybe_share_embeddings at PP=1 hands the draft the target embedding
module. Sequence parallelism shards dim 0 only, so the last dim is
hidden_size at every site.

VLLM_ABLITERATION names a JSON file with key refusal_direction, a
safetensors tensor, or a bare name resolved under
/opt/qwen38/abliteration then /data/abliteration. Empty or 0 is off.
register() always creates the buffer _abliteration_r, zeros when off, so
one compiled graph serves ON and OFF and toggling the switch
(declared in envs.py, hence in compile_factors) invalidates the compile
cache. project_() is an in-place rank-1 update with no full-size
temporary, and skips tensors whose width is not hidden_size.

Co-authored-by: Hinata <hinata@idihore.id>
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-10-15 21:40:16 +07:00
co-authored by Hinata
parent 76df77c04e
commit 0076adc78b
6 changed files with 420 additions and 2 deletions
+151
View File
@@ -0,0 +1,151 @@
# SPDX-License-Identifier: Apache-2.0
"""Optional runtime refusal-direction removal ("abliteration").
``VLLM_ABLITERATION`` (or the legacy ``QWEN38_ABLITERATION``) names a JSON file
with a ``refusal_direction`` list, or a safetensors file with that tensor, of
length ``hidden_size``. A bare name resolves to ``<name>.json`` in
``/opt/qwen38/abliteration``, then in ``/data/abliteration``. When set, every
write into the residual stream loses its component along the unit direction r:
x <- x - r (r . x)
This runs on the attention block outputs (full and linear attention), the MLP
output and the text token embeddings. For one direction it equals the weight
edit ``W' = W - r (r^T W)`` on ``self_attn.o_proj``,
``linear_attn.out_proj``, ``mlp.down_proj`` and the rows of ``embed_tokens``
(Arditi et al., 2024), so the served weights, quantized or not, stay untouched.
Unset, no projection kernel runs. The buffer always exists, holding zeros while
the switch is off, so the attribute a compiled or captured graph refers to is
always present, and the switch is part of the torch.compile cache key, so
toggling it never reuses a stale graph.
"""
import json
import math
import os
from functools import lru_cache
import torch
from vllm import envs
from vllm.logger import init_logger
logger = init_logger(__name__)
ENV = "VLLM_ABLITERATION"
BUILTIN_DIRS = ("/opt/qwen38/abliteration", "/data/abliteration")
DIRECTION = "_abliteration_r"
ENABLED = "_abliteration_on"
_shape_warned = False
def direction_path() -> str | None:
"""The configured direction file, or None when the switch is off."""
value = (envs.VLLM_ABLITERATION or "").strip()
if value in ("", "0"):
return None
if os.sep in value or value.endswith((".json", ".safetensors")):
return value
for directory in BUILTIN_DIRS:
path = os.path.join(directory, f"{value}.json")
if os.path.exists(path):
return path
return os.path.join(BUILTIN_DIRS[0], f"{value}.json")
@lru_cache(maxsize=4)
def _read(path: str) -> torch.Tensor:
"""The direction as a unit float32 CPU tensor, read once per path."""
if path.endswith(".json"):
with open(path) as f:
values = json.load(f)["refusal_direction"]
# A model built on the meta device would otherwise make this CPU tensor
# a meta one, and a meta tensor cannot be inspected or copied.
r = torch.tensor(values, dtype=torch.float32, device="cpu")
else:
from safetensors import safe_open
with safe_open(path, framework="pt", device="cpu") as f:
r = f.get_tensor("refusal_direction")
r = r.to(dtype=torch.float32, device="cpu")
r = r.flatten()
norm = float(r.norm())
if not math.isfinite(norm) or norm == 0.0:
raise ValueError(f"{ENV}: {path} holds a zero or non-finite direction")
logger.info("%s: refusal direction (%d values) from %s", ENV, r.numel(), path)
return r / norm
def load_direction(hidden_size: int) -> torch.Tensor | None:
"""The unit refusal direction on CPU, or None when the switch is off."""
path = direction_path()
if path is None:
return None
r = _read(path)
if r.numel() != hidden_size:
raise ValueError(
f"{ENV}: {path} holds {r.numel()} values; the model's hidden size "
f"is {hidden_size}"
)
return r
def register(module: torch.nn.Module, hidden_size: int, dtype: torch.dtype) -> None:
"""Attach the direction as a non-persistent buffer, from a model constructor.
Call it before CUDA-graph capture so the buffer lands on the model's device.
It holds zeros when the switch is off. A buffer that lands on the meta
device cannot hold a direction, which turns the projection off for that
module.
"""
r = load_direction(hidden_size)
buf = torch.zeros(hidden_size, dtype=dtype)
on = r is not None
if on and buf.device.type == "meta":
logger.warning(
"%s: %s is built on the meta device, which cannot hold a direction; "
"projection is off for this module",
ENV,
type(module).__name__,
)
on = False
elif on:
buf.copy_(r)
module.register_buffer(DIRECTION, buf, persistent=False)
setattr(module, ENABLED, on)
def project_(module: torch.nn.Module, x: torch.Tensor) -> torch.Tensor:
"""Remove the component of x along the direction, in place. No-op when off."""
if not getattr(module, ENABLED, False):
return x
r = getattr(module, DIRECTION)
width = x.shape[-1]
if width != r.numel():
_warn_shape(type(module).__name__, width, r.numel())
return x
if r.dtype != x.dtype:
# Activations do not always carry the model dtype, and the rank-1 update
# needs one dtype throughout.
r = r.to(x.dtype)
if x.is_contiguous():
flat = x.view(-1, width)
flat.addr_(flat @ r, r, alpha=-1.0)
else:
x.sub_(torch.outer(x.reshape(-1, width) @ r, r).view(x.shape))
return x
def _warn_shape(name: str, width: int, size: int) -> None:
global _shape_warned
if not _shape_warned:
_shape_warned = True
logger.warning(
"%s: skipped a %d-wide tensor from %s; the direction has %d values",
ENV,
width,
name,
size,
)
+3
View File
@@ -66,6 +66,7 @@ from vllm.multimodal.inputs import MultiModalFeatureSpec
from vllm.sequence import IntermediateTensors
from vllm.tokenizers.registry import cached_tokenizer_from_config
from . import abliteration
from .interfaces import (
HasInnerState,
IsHybrid,
@@ -191,6 +192,7 @@ class Qwen3_5DecoderLayer(Qwen3NextDecoderLayer):
self.post_attention_layernorm = Qwen3_5RMSNorm(
config.hidden_size, eps=config.rms_norm_eps
)
abliteration.register(self, config.hidden_size, model_config.dtype)
self.layer_scale = getattr(config, "layer_scale", False)
if self.layer_scale:
@@ -254,6 +256,7 @@ class Qwen3_5Model(Qwen3NextModel):
quant_config=self.quant_config,
prefix=f"{prefix}.embed_tokens",
)
abliteration.register(self, config.hidden_size, vllm_config.model_config.dtype)
def get_layer(prefix: str):
return Qwen3_5DecoderLayer(
+3 -1
View File
@@ -37,6 +37,7 @@ from vllm.model_executor.models.utils import sequence_parallel_chunk
from vllm.sequence import IntermediateTensors
from vllm.transformers_utils.repo_utils import try_get_local_file
from . import abliteration
from .interfaces import (
MultiModalEmbeddings,
SupportsMultiModal,
@@ -146,6 +147,7 @@ class Qwen3_5MultiTokenPredictor(nn.Module):
self.vocab_size,
config.hidden_size,
)
abliteration.register(self, config.hidden_size, model_config.dtype)
# Workaround: mtp.fc is stored as BF16 in NVFP4 checkpoints but is
# missing from hf_quant_config.json exclude_modules. Force unquantized.
@@ -217,7 +219,7 @@ class Qwen3_5MultiTokenPredictor(nn.Module):
)
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
return self.embed_tokens(input_ids)
return abliteration.project_(self, self.embed_tokens(input_ids))
def forward(
self,
+8 -1
View File
@@ -61,6 +61,7 @@ from vllm.platforms import current_platform
from vllm.sequence import IntermediateTensors
from vllm.v1.attention.backend import AttentionType
from . import abliteration
from .interfaces import (
EagleModelMixin,
HasInnerState,
@@ -537,6 +538,7 @@ class Qwen3NextDecoderLayer(nn.Module):
self.post_attention_layernorm = Qwen3NextRMSNorm(
config.hidden_size, eps=config.rms_norm_eps
)
abliteration.register(self, config.hidden_size, model_config.dtype)
self.layer_scale = getattr(config, "layer_scale", False)
if self.layer_scale:
@@ -584,6 +586,8 @@ class Qwen3NextDecoderLayer(nn.Module):
else:
raise ValueError("Invalid layer_type")
abliteration.project_(self, hidden_states)
if self.layer_scale:
if len(hidden_states.shape) == 2:
hidden_states = hidden_states * (
@@ -612,6 +616,8 @@ class Qwen3NextDecoderLayer(nn.Module):
else:
hidden_states = self.mlp(hidden_states)
abliteration.project_(self, hidden_states)
if self.layer_scale:
if len(hidden_states.shape) == 2:
hidden_states = hidden_states * (
@@ -661,6 +667,7 @@ class Qwen3NextModel(nn.Module, EagleModelMixin):
self.vocab_size,
config.hidden_size,
)
abliteration.register(self, config.hidden_size, vllm_config.model_config.dtype)
def get_layer(prefix: str):
return Qwen3NextDecoderLayer(
@@ -689,7 +696,7 @@ class Qwen3NextModel(nn.Module, EagleModelMixin):
self.aux_hidden_state_layers: tuple[int, ...] = ()
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
return self.embed_tokens(input_ids)
return abliteration.project_(self, self.embed_tokens(input_ids))
@property
def use_sequence_parallel(self) -> bool: