fix: keep the refusal-direction projection dynamo-safe
With VLLM_ABLITERATION set, boot crashed: dynamo traces abliteration.register from Qwen3NextGatedDeltaNet.__init__, and buf.copy_(r) fails on the meta device during the compile-time model construction pass. Disable tracing for register and guard the copy so an unmaterializable buffer turns projection off with a warning. Log activation at registration instead of inside project_: project_ runs in the compiled region, and a logger call there aborts dynamo. The width guard uses the same escape hatch. Also project the embedding that Qwen3NextMultiTokenPredictor builds for itself, which the first pass missed, and add tests for the 2560-value Flash-Next direction against a 5120 model and for a half-width activation. Co-authored-by: Hinata <hinata@idihore.id> Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
@@ -16,8 +16,10 @@ HIDDEN = 8
|
||||
@pytest.fixture(autouse=True)
|
||||
def clear_direction_cache():
|
||||
abliteration._read.cache_clear()
|
||||
abliteration.clear_logs()
|
||||
yield
|
||||
abliteration._read.cache_clear()
|
||||
abliteration.clear_logs()
|
||||
|
||||
|
||||
def write_direction(tmp_path, name="r.json", size=HIDDEN):
|
||||
@@ -97,6 +99,39 @@ def test_hidden_size_mismatch_raises(tmp_path, monkeypatch):
|
||||
registered(torch.nn.Module(), size=HIDDEN + 1)
|
||||
|
||||
|
||||
def test_first_projection_logs_that_it_is_active(tmp_path, monkeypatch, caplog):
|
||||
"""The boot log proves the switch is live, not just loaded."""
|
||||
import logging
|
||||
|
||||
monkeypatch.setenv("VLLM_ABLITERATION", write_direction(tmp_path))
|
||||
module = registered(torch.nn.Linear(HIDDEN, HIDDEN))
|
||||
with caplog.at_level(
|
||||
logging.INFO, logger="vllm.model_executor.models.abliteration"
|
||||
):
|
||||
abliteration.project_(module, torch.randn(2, HIDDEN))
|
||||
assert "active, projected a 8-wide tensor" in caplog.text
|
||||
|
||||
|
||||
def test_flash_next_direction_is_rejected_by_the_27b_model(tmp_path, monkeypatch):
|
||||
"""Flash-Next stores 2560 values; the dense 27B stream is 5120 wide."""
|
||||
path = write_direction(tmp_path, name="flash_next.json", size=2560)
|
||||
monkeypatch.setenv("VLLM_ABLITERATION", path)
|
||||
with pytest.raises(
|
||||
ValueError, match="holds 2560 values; the model's hidden size is 5120"
|
||||
):
|
||||
registered(torch.nn.Module(), size=5120)
|
||||
|
||||
|
||||
def test_half_width_activation_is_skipped(tmp_path, monkeypatch):
|
||||
"""A 2560-wide tensor cannot be projected by the 5120 direction."""
|
||||
monkeypatch.setenv("VLLM_ABLITERATION", write_direction(tmp_path, size=5120))
|
||||
module = registered(torch.nn.Module(), size=5120)
|
||||
x = torch.randn(4, 2560)
|
||||
before = x.clone()
|
||||
abliteration.project_(module, x)
|
||||
assert torch.equal(x, before)
|
||||
|
||||
|
||||
def test_zero_direction_raises(tmp_path, monkeypatch):
|
||||
path = tmp_path / "zero.json"
|
||||
path.write_text(json.dumps({"refusal_direction": [0.0] * HIDDEN}))
|
||||
|
||||
Reference in New Issue
Block a user