diff --git a/tests/model_executor/test_abliteration.py b/tests/model_executor/test_abliteration.py new file mode 100644 index 0000000000..ecdb6a9b61 --- /dev/null +++ b/tests/model_executor/test_abliteration.py @@ -0,0 +1,122 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Runtime abliteration equals the weight edit W' = W - r (r^T W) on CPU.""" + +import json + +import pytest +import torch + +import vllm.envs as envs +from vllm.model_executor.models import abliteration + +HIDDEN = 64 + + +@pytest.fixture +def direction(): + torch.manual_seed(0) + return torch.nn.functional.normalize( + torch.randn(HIDDEN, dtype=torch.float64), dim=0 + ) + + +@pytest.fixture +def direction_file(tmp_path, direction): + path = tmp_path / "direction.json" + path.write_text( + json.dumps({"hidden_size": HIDDEN, "refusal_direction": direction.tolist()}) + ) + return str(path) + + +def write_direction(tmp_path, values, name="d.json"): + path = tmp_path / name + path.write_text(json.dumps({"refusal_direction": values})) + return str(path) + + +def test_load_direction_normalizes_and_validates(tmp_path, direction, monkeypatch): + path = write_direction(tmp_path, (direction * 3).tolist()) + monkeypatch.setattr(envs, "VLLM_ABLITERATION", path) + r = abliteration.load_direction(HIDDEN) + assert r.dtype == torch.float32 + assert r.norm().item() == pytest.approx(1.0, abs=1e-6) + + monkeypatch.setattr(envs, "VLLM_ABLITERATION", "") + assert abliteration.load_direction(HIDDEN) is None + + monkeypatch.setattr(envs, "VLLM_ABLITERATION", path) + with pytest.raises(ValueError, match="hidden size"): + abliteration.load_direction(HIDDEN + 1) + + zero = write_direction(tmp_path, [0.0] * HIDDEN, "zero.json") + monkeypatch.setattr(envs, "VLLM_ABLITERATION", zero) + with pytest.raises(ValueError, match="zero direction"): + abliteration.load_direction(HIDDEN) + + +def test_bare_name_resolves_to_the_builtin_dir(tmp_path, monkeypatch): + (tmp_path / "orcarouter.json").write_text( + json.dumps({"refusal_direction": [1.0] + [0.0] * (HIDDEN - 1)}) + ) + monkeypatch.setattr(envs, "VLLM_ABLITERATION", "orcarouter") + monkeypatch.setenv("VLLM_ABLITERATION_DIR", str(tmp_path)) + r = abliteration.load_direction(HIDDEN) + assert r[0].item() == pytest.approx(1.0) + + monkeypatch.delenv("VLLM_ABLITERATION_DIR") + monkeypatch.setattr(envs, "VLLM_ABLITERATION", "missing") + with pytest.raises(FileNotFoundError): + abliteration.load_direction(HIDDEN) + + +def test_project_matches_weight_edit(direction): + torch.manual_seed(1) + w = torch.randn(HIDDEN, 96, dtype=torch.float64) + x = torch.randn(7, 96, dtype=torch.float64) + edited = w - torch.outer(direction, direction @ w) + y = x @ w.T + abliteration.project_(y, direction) + torch.testing.assert_close(y, x @ edited.T, rtol=1e-9, atol=1e-9) + + +def test_project_handles_strided_input(direction): + torch.manual_seed(2) + y = torch.randn(7, 128, dtype=torch.float64)[:, :HIDDEN] + reference = y - torch.outer(y @ direction, direction) + abliteration.project_(y, direction) + torch.testing.assert_close(y, reference, rtol=1e-9, atol=1e-9) + + +def test_projection_is_idempotent(direction): + torch.manual_seed(3) + x = torch.randn(4, 3, HIDDEN, dtype=torch.float64) + abliteration.project_(x, direction) + assert (x.reshape(-1, HIDDEN) @ direction).abs().max().item() < 1e-12 + once = x.clone() + abliteration.project_(x, direction) + torch.testing.assert_close(x, once, rtol=1e-12, atol=1e-12) + + +def test_projector_is_in_place_and_switchable(direction_file, monkeypatch): + monkeypatch.setattr(envs, "VLLM_ABLITERATION", direction_file) + x = torch.ones(2, HIDDEN, dtype=torch.float64) + out = abliteration.projector(HIDDEN)(x) + assert out.data_ptr() == x.data_ptr() + with open(direction_file) as f: + r = torch.tensor(json.load(f)["refusal_direction"], dtype=torch.float64) + assert (out @ r).abs().max().item() < 1e-6 + + monkeypatch.setattr(envs, "VLLM_ABLITERATION", "0") + y = torch.ones(2, HIDDEN, dtype=torch.float64) + assert abliteration.projector(HIDDEN)(y) is y + + +def test_projector_reads_env_once_per_call_site(direction_file, monkeypatch): + project = abliteration.projector(HIDDEN) + monkeypatch.setattr(envs, "VLLM_ABLITERATION", direction_file) + x = torch.ones(2, HIDDEN, dtype=torch.float64) + assert project(x).data_ptr() == x.data_ptr() + monkeypatch.setattr(envs, "VLLM_ABLITERATION", "") + assert project(x).data_ptr() == x.data_ptr() diff --git a/vllm/envs.py b/vllm/envs.py index 851741d525..0e0467510d 100755 --- a/vllm/envs.py +++ b/vllm/envs.py @@ -318,6 +318,17 @@ if TYPE_CHECKING: """If set, vLLM will skip model name validation in API requests. This allows any model name to be accepted in the 'model' field of requests, making the server model-name agnostic. Useful for proxy/gateway scenarios.""" + VLLM_ABLITERATION: str | None = None + """Switch for runtime abliteration: a path to a JSON file holding a + ``refusal_direction`` list of length ``hidden_size``, a safetensors file + holding that tensor, or a bare name resolved to + ``$VLLM_ABLITERATION_DIR/.json``. When set, that direction is + projected out of the attention, linear-attention, MLP and token-embedding + outputs, which makes a base checkpoint behave like its abliterated release. + See ``vllm/model_executor/models/abliteration.py``.""" + VLLM_ABLITERATION_DIR: str | None = None + """Directory that bare ``VLLM_ABLITERATION`` names resolve to. Defaults to + ``/opt/vllm/abliteration``.""" VLLM_ELASTIC_EP_SCALE_UP_LAUNCH: bool = False VLLM_ELASTIC_EP_DRAIN_REQUESTS: bool = False VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS: bool = True @@ -2173,6 +2184,14 @@ environment_variables: dict[str, Callable[[], Any]] = { os.getenv("VLLM_SKIP_MODEL_NAME_VALIDATION", "0").strip().lower() in ("1", "true") ), + # Switch for runtime abliteration: a JSON file with a "refusal_direction" + # list, a safetensors file with that tensor, or a bare name resolved to + # $VLLM_ABLITERATION_DIR/.json. When set, that unit direction is + # projected out of the residual-stream writes of the attention, + # linear-attention and MLP blocks, and of the token embeddings. + "VLLM_ABLITERATION": lambda: os.getenv("VLLM_ABLITERATION"), + # Directory that bare VLLM_ABLITERATION names resolve to. + "VLLM_ABLITERATION_DIR": lambda: os.getenv("VLLM_ABLITERATION_DIR"), # Whether it is a scale up launch engine for elastic EP, # Should only be set by EngineCoreClient. "VLLM_ELASTIC_EP_SCALE_UP_LAUNCH": lambda: bool( diff --git a/vllm/model_executor/models/abliteration.py b/vllm/model_executor/models/abliteration.py new file mode 100644 index 0000000000..be9eaa3066 --- /dev/null +++ b/vllm/model_executor/models/abliteration.py @@ -0,0 +1,149 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Optional refusal-direction removal ("abliteration"), applied at runtime. + +``VLLM_ABLITERATION`` names a JSON file holding a ``refusal_direction`` list, +or a safetensors file holding a ``refusal_direction`` tensor, of length +``hidden_size``. A bare name such as ``orcarouter`` resolves to +``$VLLM_ABLITERATION_DIR/.json``, defaulting to +``/opt/vllm/abliteration/.json``. With it set, every residual-stream +write loses its component along the unit direction r: + + x <- x - r (r . x) + +It is applied to the attention, linear-attention and MLP block outputs, and to +the text token embeddings. For one direction that equals the weight edit +``W' = W - r (r^T W)`` on ``self_attn.o_proj``, ``linear_attn.out_proj``, +``mlp.down_proj`` and the rows of ``embed_tokens`` (Arditi et al., 2024), so a +base checkpoint behaves like its abliterated release with no extra weights. +The Flash-Next runtime overlay uses the same rule. Unset, nothing loads and no +work is added. +""" + +import functools +import json +import math +import os + +import torch + +from vllm.distributed import ( + get_tensor_model_parallel_rank, + get_tensor_model_parallel_world_size, +) +from vllm.logger import init_logger + +logger = init_logger(__name__) + +ENV = "VLLM_ABLITERATION" +BUILTIN_DIR = "/opt/vllm/abliteration" + + +def direction_path() -> str | None: + """The direction file to load, None when the switch is off.""" + from vllm.envs import VLLM_ABLITERATION, VLLM_ABLITERATION_DIR + + value = (VLLM_ABLITERATION or "").strip() + if value in ("", "0"): + return None + if os.sep not in value and not value.endswith((".json", ".safetensors")): + root = (VLLM_ABLITERATION_DIR or "").strip() or BUILTIN_DIR + value = os.path.join(root, f"{value}.json") + return value + + +@functools.lru_cache(maxsize=8) +def _load_unit_direction(path: str, hidden_size: int) -> torch.Tensor: + if path.endswith(".safetensors"): + from safetensors import safe_open + + with safe_open(path, framework="pt", device="cpu") as f: + r = f.get_tensor("refusal_direction") + else: + with open(path) as f: + r = torch.tensor(json.load(f)["refusal_direction"], dtype=torch.float32) + r = r.to(torch.float32).flatten() + if r.numel() != hidden_size: + raise ValueError( + f"{ENV}: {path} holds {r.numel()} values; " + f"the model's hidden size is {hidden_size}" + ) + norm = float(r.norm()) + if not math.isfinite(norm) or norm == 0.0: + raise ValueError(f"{ENV}: {path} holds a zero direction") + return r / norm + + +def load_direction(hidden_size: int) -> torch.Tensor | None: + """The unit refusal direction as float32 on CPU, None when the switch is off.""" + path = direction_path() + if path is None: + return None + r = _load_unit_direction(path, hidden_size) + logger.info_once( + "Abliteration: removing the refusal direction in %s (%d values).", + path, + hidden_size, + ) + return r + + +def _slice_for(r: torch.Tensor, width: int) -> torch.Tensor: + """r, or the slice of it that a width-wide tensor covers.""" + if width == r.shape[-1]: + return r + tp = get_tensor_model_parallel_world_size() + if width * tp != r.shape[-1]: + raise ValueError( + f"abliteration: width {width} matches neither the hidden size " + f"{r.shape[-1]} nor its {tp}-way shard" + ) + return r.narrow(0, get_tensor_model_parallel_rank() * width, width) + + +def _ready(r: torch.Tensor, x: torch.Tensor) -> torch.Tensor: + """r, sliced and cast for x. The cast result is cached on the shared copy.""" + width = x.shape[-1] + copies = r.__dict__.setdefault("_vllm_copies", {}) + key = (x.device, x.dtype, width) + part = copies.get(key) + if part is None: + part = _slice_for(r, width).to(device=x.device, dtype=x.dtype) + copies[key] = part + return part + + +def project_(x: torch.Tensor, r: torch.Tensor | None) -> torch.Tensor: + """In place: drop the component of x along r. No-op when r is None.""" + if r is None: + return x + width = x.shape[-1] + part = _ready(r, x) + if x.is_contiguous(): + flat = x.view(-1, width) + # rank-1 update, no full-size temporary + flat.addr_(flat @ part, part, alpha=-1.0) + else: + x.sub_(part * x.matmul(part).unsqueeze(-1)) + return x + + +def projector(hidden_size: int): + """Return ``project(x)``: in-place projection of x, a no-op when switched off. + + Model code stores the callable on the module, so torch.compile treats it as + an opaque custom op, and no device-dependent tensor is built at construction. + """ + state: dict[str, object] = {} + + def direction() -> torch.Tensor | None: + if "loaded" not in state: + state["loaded"] = True + state["r"] = load_direction(hidden_size) + return state["r"] + + def project(x: torch.Tensor) -> torch.Tensor: + r = direction() + return x if r is None else project_(x, r) + + return project diff --git a/vllm/model_executor/models/qwen3_5.py b/vllm/model_executor/models/qwen3_5.py index 5b72132160..fb9724db08 100644 --- a/vllm/model_executor/models/qwen3_5.py +++ b/vllm/model_executor/models/qwen3_5.py @@ -66,6 +66,7 @@ from vllm.multimodal.inputs import MultiModalFeatureSpec from vllm.sequence import IntermediateTensors from vllm.tokenizers.registry import cached_tokenizer_from_config +from .abliteration import projector from .interfaces import ( HasInnerState, IsHybrid, @@ -191,6 +192,7 @@ class Qwen3_5DecoderLayer(Qwen3NextDecoderLayer): ) self.layer_scale = getattr(config, "layer_scale", False) + self.abliterate = projector(config.hidden_size) if self.layer_scale: self.attn_layer_scale = torch.nn.Parameter( torch.zeros( diff --git a/vllm/model_executor/models/qwen3_5_mtp.py b/vllm/model_executor/models/qwen3_5_mtp.py index 763c6a5eb6..beae3886cd 100644 --- a/vllm/model_executor/models/qwen3_5_mtp.py +++ b/vllm/model_executor/models/qwen3_5_mtp.py @@ -23,6 +23,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import ( ParallelLMHead, VocabParallelEmbedding, ) +from vllm.model_executor.models import abliteration from vllm.model_executor.models.interfaces import LocalArgmaxMixin from vllm.model_executor.models.qwen3_5 import ( Qwen3_5DecoderLayer, @@ -215,9 +216,10 @@ class Qwen3_5MultiTokenPredictor(nn.Module): self.pre_fc_norm_embedding = Qwen3_5RMSNorm( config.hidden_size, eps=config.rms_norm_eps ) + self.abliterate = abliteration.projector(config.hidden_size) def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor: - return self.embed_tokens(input_ids) + return self.abliterate(self.embed_tokens(input_ids)) def forward( self, diff --git a/vllm/model_executor/models/qwen3_next.py b/vllm/model_executor/models/qwen3_next.py index b0bf0fecd1..fc1e4359fc 100644 --- a/vllm/model_executor/models/qwen3_next.py +++ b/vllm/model_executor/models/qwen3_next.py @@ -61,6 +61,7 @@ from vllm.platforms import current_platform from vllm.sequence import IntermediateTensors from vllm.v1.attention.backend import AttentionType +from .abliteration import projector from .interfaces import ( EagleModelMixin, HasInnerState, @@ -531,6 +532,7 @@ class Qwen3NextDecoderLayer(nn.Module): ) self.layer_scale = getattr(config, "layer_scale", False) + self.abliterate = projector(config.hidden_size) if self.layer_scale: self.attn_layer_scale = torch.nn.Parameter( torch.zeros( @@ -575,6 +577,7 @@ class Qwen3NextDecoderLayer(nn.Module): ) else: raise ValueError("Invalid layer_type") + hidden_states = self.abliterate(hidden_states) if self.layer_scale: if len(hidden_states.shape) == 2: @@ -603,6 +606,7 @@ class Qwen3NextDecoderLayer(nn.Module): ) else: hidden_states = self.mlp(hidden_states) + hidden_states = self.abliterate(hidden_states) if self.layer_scale: if len(hidden_states.shape) == 2: @@ -679,9 +683,10 @@ class Qwen3NextModel(nn.Module, EagleModelMixin): self.norm = PPMissingLayer() self.aux_hidden_state_layers: tuple[int, ...] = () + self.abliterate = projector(config.hidden_size) def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor: - return self.embed_tokens(input_ids) + return self.abliterate(self.embed_tokens(input_ids)) @property def use_sequence_parallel(self) -> bool: