[Model] Add opt-in runtime abliteration for Qwen3.5/Qwen3.8

Project a unit refusal direction out of the residual-stream writes
(attention, linear-attention and MLP outputs, and token embeddings) at
runtime, enabled by VLLM_ABLITERATION pointing to a JSON or safetensors
file with a refusal_direction vector. A bare name resolves to
VLLM_ABLITERATION_DIR, default /opt/vllm/abliteration. For one direction
this equals the weight edit W' = W - r(r^T W), so a base checkpoint
reproduces its abliterated release with no extra weights. Unset, the
path loads nothing and adds no work.

Co-Authored-By: Hinata <hinata@idihore.id>
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-10-14 07:57:59 +00:00
co-authored by Hinata
parent ac305c44cb
commit 8151c7b158
6 changed files with 301 additions and 2 deletions
+122
View File
@@ -0,0 +1,122 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Runtime abliteration equals the weight edit W' = W - r (r^T W) on CPU."""
import json
import pytest
import torch
import vllm.envs as envs
from vllm.model_executor.models import abliteration
HIDDEN = 64
@pytest.fixture
def direction():
torch.manual_seed(0)
return torch.nn.functional.normalize(
torch.randn(HIDDEN, dtype=torch.float64), dim=0
)
@pytest.fixture
def direction_file(tmp_path, direction):
path = tmp_path / "direction.json"
path.write_text(
json.dumps({"hidden_size": HIDDEN, "refusal_direction": direction.tolist()})
)
return str(path)
def write_direction(tmp_path, values, name="d.json"):
path = tmp_path / name
path.write_text(json.dumps({"refusal_direction": values}))
return str(path)
def test_load_direction_normalizes_and_validates(tmp_path, direction, monkeypatch):
path = write_direction(tmp_path, (direction * 3).tolist())
monkeypatch.setattr(envs, "VLLM_ABLITERATION", path)
r = abliteration.load_direction(HIDDEN)
assert r.dtype == torch.float32
assert r.norm().item() == pytest.approx(1.0, abs=1e-6)
monkeypatch.setattr(envs, "VLLM_ABLITERATION", "")
assert abliteration.load_direction(HIDDEN) is None
monkeypatch.setattr(envs, "VLLM_ABLITERATION", path)
with pytest.raises(ValueError, match="hidden size"):
abliteration.load_direction(HIDDEN + 1)
zero = write_direction(tmp_path, [0.0] * HIDDEN, "zero.json")
monkeypatch.setattr(envs, "VLLM_ABLITERATION", zero)
with pytest.raises(ValueError, match="zero direction"):
abliteration.load_direction(HIDDEN)
def test_bare_name_resolves_to_the_builtin_dir(tmp_path, monkeypatch):
(tmp_path / "orcarouter.json").write_text(
json.dumps({"refusal_direction": [1.0] + [0.0] * (HIDDEN - 1)})
)
monkeypatch.setattr(envs, "VLLM_ABLITERATION", "orcarouter")
monkeypatch.setenv("VLLM_ABLITERATION_DIR", str(tmp_path))
r = abliteration.load_direction(HIDDEN)
assert r[0].item() == pytest.approx(1.0)
monkeypatch.delenv("VLLM_ABLITERATION_DIR")
monkeypatch.setattr(envs, "VLLM_ABLITERATION", "missing")
with pytest.raises(FileNotFoundError):
abliteration.load_direction(HIDDEN)
def test_project_matches_weight_edit(direction):
torch.manual_seed(1)
w = torch.randn(HIDDEN, 96, dtype=torch.float64)
x = torch.randn(7, 96, dtype=torch.float64)
edited = w - torch.outer(direction, direction @ w)
y = x @ w.T
abliteration.project_(y, direction)
torch.testing.assert_close(y, x @ edited.T, rtol=1e-9, atol=1e-9)
def test_project_handles_strided_input(direction):
torch.manual_seed(2)
y = torch.randn(7, 128, dtype=torch.float64)[:, :HIDDEN]
reference = y - torch.outer(y @ direction, direction)
abliteration.project_(y, direction)
torch.testing.assert_close(y, reference, rtol=1e-9, atol=1e-9)
def test_projection_is_idempotent(direction):
torch.manual_seed(3)
x = torch.randn(4, 3, HIDDEN, dtype=torch.float64)
abliteration.project_(x, direction)
assert (x.reshape(-1, HIDDEN) @ direction).abs().max().item() < 1e-12
once = x.clone()
abliteration.project_(x, direction)
torch.testing.assert_close(x, once, rtol=1e-12, atol=1e-12)
def test_projector_is_in_place_and_switchable(direction_file, monkeypatch):
monkeypatch.setattr(envs, "VLLM_ABLITERATION", direction_file)
x = torch.ones(2, HIDDEN, dtype=torch.float64)
out = abliteration.projector(HIDDEN)(x)
assert out.data_ptr() == x.data_ptr()
with open(direction_file) as f:
r = torch.tensor(json.load(f)["refusal_direction"], dtype=torch.float64)
assert (out @ r).abs().max().item() < 1e-6
monkeypatch.setattr(envs, "VLLM_ABLITERATION", "0")
y = torch.ones(2, HIDDEN, dtype=torch.float64)
assert abliteration.projector(HIDDEN)(y) is y
def test_projector_reads_env_once_per_call_site(direction_file, monkeypatch):
project = abliteration.projector(HIDDEN)
monkeypatch.setattr(envs, "VLLM_ABLITERATION", direction_file)
x = torch.ones(2, HIDDEN, dtype=torch.float64)
assert project(x).data_ptr() == x.data_ptr()
monkeypatch.setattr(envs, "VLLM_ABLITERATION", "")
assert project(x).data_ptr() == x.data_ptr()
+19
View File
@@ -318,6 +318,17 @@ if TYPE_CHECKING:
"""If set, vLLM will skip model name validation in API requests.
This allows any model name to be accepted in the 'model' field of requests,
making the server model-name agnostic. Useful for proxy/gateway scenarios."""
VLLM_ABLITERATION: str | None = None
"""Switch for runtime abliteration: a path to a JSON file holding a
``refusal_direction`` list of length ``hidden_size``, a safetensors file
holding that tensor, or a bare name resolved to
``$VLLM_ABLITERATION_DIR/<name>.json``. When set, that direction is
projected out of the attention, linear-attention, MLP and token-embedding
outputs, which makes a base checkpoint behave like its abliterated release.
See ``vllm/model_executor/models/abliteration.py``."""
VLLM_ABLITERATION_DIR: str | None = None
"""Directory that bare ``VLLM_ABLITERATION`` names resolve to. Defaults to
``/opt/vllm/abliteration``."""
VLLM_ELASTIC_EP_SCALE_UP_LAUNCH: bool = False
VLLM_ELASTIC_EP_DRAIN_REQUESTS: bool = False
VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS: bool = True
@@ -2173,6 +2184,14 @@ environment_variables: dict[str, Callable[[], Any]] = {
os.getenv("VLLM_SKIP_MODEL_NAME_VALIDATION", "0").strip().lower()
in ("1", "true")
),
# Switch for runtime abliteration: a JSON file with a "refusal_direction"
# list, a safetensors file with that tensor, or a bare name resolved to
# $VLLM_ABLITERATION_DIR/<name>.json. When set, that unit direction is
# projected out of the residual-stream writes of the attention,
# linear-attention and MLP blocks, and of the token embeddings.
"VLLM_ABLITERATION": lambda: os.getenv("VLLM_ABLITERATION"),
# Directory that bare VLLM_ABLITERATION names resolve to.
"VLLM_ABLITERATION_DIR": lambda: os.getenv("VLLM_ABLITERATION_DIR"),
# Whether it is a scale up launch engine for elastic EP,
# Should only be set by EngineCoreClient.
"VLLM_ELASTIC_EP_SCALE_UP_LAUNCH": lambda: bool(
+149
View File
@@ -0,0 +1,149 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Optional refusal-direction removal ("abliteration"), applied at runtime.
``VLLM_ABLITERATION`` names a JSON file holding a ``refusal_direction`` list,
or a safetensors file holding a ``refusal_direction`` tensor, of length
``hidden_size``. A bare name such as ``orcarouter`` resolves to
``$VLLM_ABLITERATION_DIR/<name>.json``, defaulting to
``/opt/vllm/abliteration/<name>.json``. With it set, every residual-stream
write loses its component along the unit direction r:
x <- x - r (r . x)
It is applied to the attention, linear-attention and MLP block outputs, and to
the text token embeddings. For one direction that equals the weight edit
``W' = W - r (r^T W)`` on ``self_attn.o_proj``, ``linear_attn.out_proj``,
``mlp.down_proj`` and the rows of ``embed_tokens`` (Arditi et al., 2024), so a
base checkpoint behaves like its abliterated release with no extra weights.
The Flash-Next runtime overlay uses the same rule. Unset, nothing loads and no
work is added.
"""
import functools
import json
import math
import os
import torch
from vllm.distributed import (
get_tensor_model_parallel_rank,
get_tensor_model_parallel_world_size,
)
from vllm.logger import init_logger
logger = init_logger(__name__)
ENV = "VLLM_ABLITERATION"
BUILTIN_DIR = "/opt/vllm/abliteration"
def direction_path() -> str | None:
"""The direction file to load, None when the switch is off."""
from vllm.envs import VLLM_ABLITERATION, VLLM_ABLITERATION_DIR
value = (VLLM_ABLITERATION or "").strip()
if value in ("", "0"):
return None
if os.sep not in value and not value.endswith((".json", ".safetensors")):
root = (VLLM_ABLITERATION_DIR or "").strip() or BUILTIN_DIR
value = os.path.join(root, f"{value}.json")
return value
@functools.lru_cache(maxsize=8)
def _load_unit_direction(path: str, hidden_size: int) -> torch.Tensor:
if path.endswith(".safetensors"):
from safetensors import safe_open
with safe_open(path, framework="pt", device="cpu") as f:
r = f.get_tensor("refusal_direction")
else:
with open(path) as f:
r = torch.tensor(json.load(f)["refusal_direction"], dtype=torch.float32)
r = r.to(torch.float32).flatten()
if r.numel() != hidden_size:
raise ValueError(
f"{ENV}: {path} holds {r.numel()} values; "
f"the model's hidden size is {hidden_size}"
)
norm = float(r.norm())
if not math.isfinite(norm) or norm == 0.0:
raise ValueError(f"{ENV}: {path} holds a zero direction")
return r / norm
def load_direction(hidden_size: int) -> torch.Tensor | None:
"""The unit refusal direction as float32 on CPU, None when the switch is off."""
path = direction_path()
if path is None:
return None
r = _load_unit_direction(path, hidden_size)
logger.info_once(
"Abliteration: removing the refusal direction in %s (%d values).",
path,
hidden_size,
)
return r
def _slice_for(r: torch.Tensor, width: int) -> torch.Tensor:
"""r, or the slice of it that a width-wide tensor covers."""
if width == r.shape[-1]:
return r
tp = get_tensor_model_parallel_world_size()
if width * tp != r.shape[-1]:
raise ValueError(
f"abliteration: width {width} matches neither the hidden size "
f"{r.shape[-1]} nor its {tp}-way shard"
)
return r.narrow(0, get_tensor_model_parallel_rank() * width, width)
def _ready(r: torch.Tensor, x: torch.Tensor) -> torch.Tensor:
"""r, sliced and cast for x. The cast result is cached on the shared copy."""
width = x.shape[-1]
copies = r.__dict__.setdefault("_vllm_copies", {})
key = (x.device, x.dtype, width)
part = copies.get(key)
if part is None:
part = _slice_for(r, width).to(device=x.device, dtype=x.dtype)
copies[key] = part
return part
def project_(x: torch.Tensor, r: torch.Tensor | None) -> torch.Tensor:
"""In place: drop the component of x along r. No-op when r is None."""
if r is None:
return x
width = x.shape[-1]
part = _ready(r, x)
if x.is_contiguous():
flat = x.view(-1, width)
# rank-1 update, no full-size temporary
flat.addr_(flat @ part, part, alpha=-1.0)
else:
x.sub_(part * x.matmul(part).unsqueeze(-1))
return x
def projector(hidden_size: int):
"""Return ``project(x)``: in-place projection of x, a no-op when switched off.
Model code stores the callable on the module, so torch.compile treats it as
an opaque custom op, and no device-dependent tensor is built at construction.
"""
state: dict[str, object] = {}
def direction() -> torch.Tensor | None:
if "loaded" not in state:
state["loaded"] = True
state["r"] = load_direction(hidden_size)
return state["r"]
def project(x: torch.Tensor) -> torch.Tensor:
r = direction()
return x if r is None else project_(x, r)
return project
+2
View File
@@ -66,6 +66,7 @@ from vllm.multimodal.inputs import MultiModalFeatureSpec
from vllm.sequence import IntermediateTensors
from vllm.tokenizers.registry import cached_tokenizer_from_config
from .abliteration import projector
from .interfaces import (
HasInnerState,
IsHybrid,
@@ -191,6 +192,7 @@ class Qwen3_5DecoderLayer(Qwen3NextDecoderLayer):
)
self.layer_scale = getattr(config, "layer_scale", False)
self.abliterate = projector(config.hidden_size)
if self.layer_scale:
self.attn_layer_scale = torch.nn.Parameter(
torch.zeros(
+3 -1
View File
@@ -23,6 +23,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
ParallelLMHead,
VocabParallelEmbedding,
)
from vllm.model_executor.models import abliteration
from vllm.model_executor.models.interfaces import LocalArgmaxMixin
from vllm.model_executor.models.qwen3_5 import (
Qwen3_5DecoderLayer,
@@ -215,9 +216,10 @@ class Qwen3_5MultiTokenPredictor(nn.Module):
self.pre_fc_norm_embedding = Qwen3_5RMSNorm(
config.hidden_size, eps=config.rms_norm_eps
)
self.abliterate = abliteration.projector(config.hidden_size)
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
return self.embed_tokens(input_ids)
return self.abliterate(self.embed_tokens(input_ids))
def forward(
self,
+6 -1
View File
@@ -61,6 +61,7 @@ from vllm.platforms import current_platform
from vllm.sequence import IntermediateTensors
from vllm.v1.attention.backend import AttentionType
from .abliteration import projector
from .interfaces import (
EagleModelMixin,
HasInnerState,
@@ -531,6 +532,7 @@ class Qwen3NextDecoderLayer(nn.Module):
)
self.layer_scale = getattr(config, "layer_scale", False)
self.abliterate = projector(config.hidden_size)
if self.layer_scale:
self.attn_layer_scale = torch.nn.Parameter(
torch.zeros(
@@ -575,6 +577,7 @@ class Qwen3NextDecoderLayer(nn.Module):
)
else:
raise ValueError("Invalid layer_type")
hidden_states = self.abliterate(hidden_states)
if self.layer_scale:
if len(hidden_states.shape) == 2:
@@ -603,6 +606,7 @@ class Qwen3NextDecoderLayer(nn.Module):
)
else:
hidden_states = self.mlp(hidden_states)
hidden_states = self.abliterate(hidden_states)
if self.layer_scale:
if len(hidden_states.shape) == 2:
@@ -679,9 +683,10 @@ class Qwen3NextModel(nn.Module, EagleModelMixin):
self.norm = PPMissingLayer()
self.aux_hidden_state_layers: tuple[int, ...] = ()
self.abliterate = projector(config.hidden_size)
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
return self.embed_tokens(input_ids)
return self.abliterate(self.embed_tokens(input_ids))
@property
def use_sequence_parallel(self) -> bool: