[Mini-VLLM] Cut dead subsystems across serving, connectors, parsers

- Remove P/D transfer connectors: nixl, nixl_chat, mooncake x2, hf3fs,
  moriio, hisparse, example and hidden-states connectors
- Reduce weight_transfer to the base API used by model loading
- Remove kv_offload tiering, simple_kv_offload, ec cpu/example,
  snapshot command, engine snapshot server
- Remove transformers model fallback, dsv4 tokenizers, dead platforms
  cpu/tpu/xpu, flashmla third_party, dsa, megamoe/flydsl, dead attn
  ops, dead layers (short_conv, fused_embed_norm, indexer_topk,
  relu2 gemma_ar, lightning_attn, minimax_rms_norm), gemm_rs_ar
- Keep MoE LoRA op (imported eagerly by punica_gpu); drop fp8 variant
- Rebuild parser hubs around Qwen3 adapters only (36 dead adapters)
- Remove dead tests with the code they covered

Co-authored-by: Hinata <hinata@idihore.id>
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-10-19 02:19:03 +07:00
co-authored by Hinata
parent 54a396586e
commit 2b544b7ba0
353 changed files with 29 additions and 121269 deletions
@@ -250,8 +250,8 @@ def deepseekvl2_patch_hf_runner(hf_model: HfRunner) -> HfRunner:
return BatchFeature(data=inputs, tensor_type="pt")
hf_model.processor = processor
hf_model.model.get_output_embeddings = (
lambda: hf_model.model.language.model.embed_tokens
hf_model.model.get_output_embeddings = lambda: (
hf_model.model.language.model.embed_tokens
)
return hf_model
@@ -353,8 +353,8 @@ def glm4v_patch_hf_runner(hf_model: HfRunner) -> HfRunner:
)
hf_model.processor = processor
hf_model.model.get_output_embeddings = (
lambda: hf_model.model.transformer.output_layer
hf_model.model.get_output_embeddings = lambda: (
hf_model.model.transformer.output_layer
)
return hf_model
@@ -442,8 +442,8 @@ def h2ovl_patch_hf_runner(hf_model: HfRunner) -> HfRunner:
img_context_token_id = hf_model.tokenizer.convert_tokens_to_ids("<IMG_CONTEXT>")
hf_model.model.img_context_token_id = img_context_token_id
hf_model.processor = H2OVLProcessor(hf_model)
hf_model.model.get_output_embeddings = (
lambda: hf_model.model.language_model.get_output_embeddings()
hf_model.model.get_output_embeddings = lambda: (
hf_model.model.language_model.get_output_embeddings()
)
hf_model.model.generate = types.MethodType(_internvl_generate, hf_model.model)
return hf_model
@@ -678,8 +678,8 @@ def skyworkr1v_patch_hf_runner(hf_model: HfRunner) -> HfRunner:
img_context_token_id = hf_model.tokenizer.convert_tokens_to_ids("<IMG_CONTEXT>")
hf_model.model.img_context_token_id = img_context_token_id
hf_model.processor = SkyworkR1VProcessor(hf_model)
hf_model.model.get_output_embeddings = (
lambda: hf_model.model.language_model.get_output_embeddings()
hf_model.model.get_output_embeddings = lambda: (
hf_model.model.language_model.get_output_embeddings()
)
hf_model.model.generate = types.MethodType(_internvl_generate, hf_model.model)
return hf_model
@@ -787,8 +787,8 @@ def internvl_patch_hf_runner(hf_model: HfRunner) -> HfRunner:
img_context_token_id = hf_model.tokenizer.convert_tokens_to_ids("<IMG_CONTEXT>")
hf_model.model.img_context_token_id = img_context_token_id
hf_model.processor = InternVLProcessor(hf_model)
hf_model.model.get_output_embeddings = (
lambda: hf_model.model.language_model.get_output_embeddings()
hf_model.model.get_output_embeddings = lambda: (
hf_model.model.language_model.get_output_embeddings()
)
hf_model.model.generate = types.MethodType(_internvl_generate, hf_model.model)
return hf_model
@@ -941,8 +941,8 @@ def molmo_patch_hf_runner(hf_model: HfRunner) -> HfRunner:
def ovis_patch_hf_runner(hf_model: HfRunner) -> HfRunner:
"""Patches and returns an instance of the HfRunner to use for Ovis2."""
hf_model.model.get_output_embeddings = (
lambda: hf_model.model.llm.get_output_embeddings()
hf_model.model.get_output_embeddings = lambda: (
hf_model.model.llm.get_output_embeddings()
)
def processor(*args, text="", images=None, **kwargs):
@@ -977,8 +977,8 @@ def ovis_patch_hf_runner(hf_model: HfRunner) -> HfRunner:
def ovis2_5_patch_hf_runner(hf_model: HfRunner) -> HfRunner:
"""Patches and returns an instance of the HfRunner to use for Ovis2."""
hf_model.model.get_output_embeddings = (
lambda: hf_model.model.llm.get_output_embeddings()
hf_model.model.get_output_embeddings = lambda: (
hf_model.model.llm.get_output_embeddings()
)
def processor(*args, text="", images=None, videos=None, **kwargs):
@@ -1209,221 +1209,3 @@ def voxtral_patch_hf_runner(hf_model: "HfRunner") -> "HfRunner":
hf_model.get_inputs = patched_get_inputs # type: ignore[method-assign]
hf_model.model.generate = patched_generate # type: ignore[method-assign]
return hf_model
def moondream3_processor(model: str):
from vllm.transformers_utils.processors.moondream3 import Moondream3Processor
return Moondream3Processor.from_pretrained(model, trust_remote_code=True)
def moondream3_patch_hf_runner(hf_model: HfRunner) -> HfRunner:
"""Patch HfRunner for Moondream3."""
moondream_processor = hf_model.processor
def processor(*args, text="", images=None, **kwargs):
if images is None:
return moondream_processor(text=text, **kwargs)
images_list = [images] if isinstance(images, Image) else images
return moondream_processor(images=images_list, text=text, **kwargs)
hf_model.processor = processor
# Expose the LM head for logprob extraction.
hf_model.model.get_output_embeddings = lambda: hf_model.model.model.text.lm_head
native_model = hf_model.model.model # MoondreamModel instance
from torch.nn import functional as F
from vllm.model_executor.models.moondream3 import reconstruct_from_crops
# Resolve the placeholder tokens from the tokenizer instead of hard-coding.
image_placeholder_ids = moondream_processor.tokenizer.encode(
"<image>", add_special_tokens=False
)
def _normalize_tiling(tilings):
"""Extract (h, w) tuple from various tiling container formats."""
tiling = tilings
if isinstance(tiling, torch.Tensor):
tiling = tuple(tiling.squeeze().tolist())
elif isinstance(tiling, (list, tuple)):
t0 = tiling[0]
if isinstance(t0, torch.Tensor):
tiling = tuple(t0.tolist())
elif isinstance(t0, (list, tuple)):
tiling = tuple(t0)
return tiling
def _encode_vision(pixel_values, tilings):
"""Run preprocessed crops through vision encoder + projection."""
device = native_model.device
dtype = native_model.vision.pos_emb.dtype
config = native_model.config
pv = pixel_values
while pv.dim() > 4:
pv = pv.squeeze(0)
pv = pv.to(device=device, dtype=dtype)
features = native_model._vis_enc(pv)
grid_size = config.vision.crop_size // config.vision.enc_patch_size
global_feat = features[0]
if features.shape[0] > 1 and tilings is not None:
tiling = _normalize_tiling(tilings)
local = features[1:].view(-1, grid_size, grid_size, config.vision.enc_dim)
reconstructed = reconstruct_from_crops(
local,
tiling,
config.vision.overlap_margin,
patch_size=1,
)
else:
reconstructed = global_feat.view(
grid_size, grid_size, config.vision.enc_dim
)
return native_model._vis_proj(global_feat, reconstructed)
def _find_subsequence(seq, subseq):
"""Find start index of subseq in seq, or None."""
n = len(subseq)
for i in range(len(seq) - n + 1):
if seq[i : i + n] == subseq:
return i
return None
def _generate(
self,
input_ids=None,
pixel_values=None,
tilings=None,
attention_mask=None,
**kwargs,
):
max_new_tokens = kwargs.get("max_new_tokens", 128)
return_dict = kwargs.get("return_dict_in_generate", False)
output_hs = kwargs.get("output_hidden_states", False)
if pixel_values is None:
sequences = input_ids
if return_dict:
return types.SimpleNamespace(
sequences=sequences,
hidden_states=() if output_hs else None,
)
return sequences
# Processor may return lists; extract the single element.
if isinstance(pixel_values, (list, tuple)):
pixel_values = pixel_values[0]
if (
isinstance(tilings, (list, tuple))
and tilings
and not isinstance(tilings[0], int)
):
tilings = tilings[0]
hf_model.model._setup_caches()
native_model.use_flex_decoding = False
device = native_model.device
config = native_model.config
with torch.inference_mode():
for block in native_model.text.blocks:
block.kv_cache.k_cache.zero_()
block.kv_cache.v_cache.zero_()
img_emb = _encode_vision(pixel_values, tilings)
bos_emb = F.embedding(
torch.tensor([[config.tokenizer.bos_id]], device=device),
native_model.text.wte,
)
img_input = torch.cat([bos_emb, img_emb.unsqueeze(0)], dim=1)
prefix_len = img_input.size(1)
mask = native_model.attn_mask[:, :, :prefix_len, :]
pos_ids = torch.arange(prefix_len, dtype=torch.long, device=device)
native_model._prefill(img_input, mask, pos_ids, None)
ids = input_ids.squeeze(0).tolist()
img_start = _find_subsequence(ids, image_placeholder_ids)
if img_start is None:
sequences = input_ids
if return_dict:
return types.SimpleNamespace(
sequences=sequences,
hidden_states=() if output_hs else None,
)
return sequences
prompt_tokens = ids[img_start + len(image_placeholder_ids) :]
if not prompt_tokens:
sequences = input_ids
if return_dict:
return types.SimpleNamespace(
sequences=sequences,
hidden_states=() if output_hs else None,
)
return sequences
prompt_tensor = torch.tensor([prompt_tokens], device=device)
prompt_emb = F.embedding(prompt_tensor, native_model.text.wte)
prompt_len = prompt_emb.size(1)
mask = native_model.attn_mask[:, :, prefix_len : prefix_len + prompt_len, :]
pos_ids = torch.arange(
prefix_len,
prefix_len + prompt_len,
dtype=torch.long,
device=device,
)
hidden = native_model._prefill(prompt_emb, mask, pos_ids, None)
pos = prefix_len + prompt_len
hidden_last = native_model.text.post_ln(hidden[:, -1:, :])
logits = native_model.text.lm_head(hidden_last.squeeze(1))
generated = []
all_hidden_states = []
# Record the hidden state that predicted each generated token.
prev_hs = hidden_last
for _ in range(max_new_tokens):
next_token = logits.argmax(dim=-1).item()
if next_token == 0:
break
generated.append(next_token)
if output_hs:
all_hidden_states.append((prev_hs,))
next_emb = F.embedding(
torch.tensor([[next_token]], device=device),
native_model.text.wte,
)
mask = native_model.attn_mask[:, :, pos : pos + 1, :]
pos_ids_step = torch.tensor([pos], dtype=torch.long, device=device)
hidden = native_model._prefill(next_emb, mask, pos_ids_step, None)
hidden_last = native_model.text.post_ln(hidden[:, -1:, :])
prev_hs = hidden_last
logits = native_model.text.lm_head(hidden_last.squeeze(1))
pos += 1
result_ids = ids + generated
sequences = torch.tensor([result_ids], device=device)
if return_dict:
return types.SimpleNamespace(
sequences=sequences,
hidden_states=tuple(all_hidden_states) if output_hs else None,
)
return sequences
hf_model.model.generate = types.MethodType(_generate, hf_model.model)
return hf_model
-436
View File
@@ -15,7 +15,6 @@ from safetensors import safe_open
from safetensors.torch import load, load_file, save_file
from torch import nn
from torch.nn import functional as F
from transformers import Qwen4ExpTextConfig
import vllm.model_executor.layers.vocab_parallel_embedding as embedding_module
import vllm.model_executor.parameter as parameter_module
@@ -31,10 +30,6 @@ from vllm.model_executor.layers.quantization.modelopt import (
ModelOptNvFp4Config,
)
from vllm.model_executor.layers.quantization.online.base import OnlineQuantizationConfig
from vllm.models.qwen4_exp.amd import ple_layer as amd_ple_layer
from vllm.models.qwen4_exp.amd.ple_layer import (
Qwen4ExpPLELayer as Qwen4ExpPLELayerAMD,
)
from vllm.models.qwen4_exp.common.ple import (
PLEShardOverlap,
compute_ple_shard_overlap,
@@ -51,7 +46,6 @@ from vllm.models.qwen4_exp.nvidia.ngram_embedding import (
Qwen4ExpPLEUnquantizedEmbeddingMethod,
)
from vllm.models.qwen4_exp.nvidia.ple_layer import Qwen4ExpPLELayer
from vllm.platforms import current_platform
from vllm.utils.torch_utils import weak_ref_tensor
from vllm.v1.attention.backends.short_conv_attn import (
PleShortConvAttentionMetadata,
@@ -732,10 +726,6 @@ def test_ple_embedding_respects_modelopt_exclusion() -> None:
)
def test_ple_embedding_dtype_overrides_modelopt_exclusion() -> None:
prefix = "model.layers.1.ple.ple_embedding.ngram_embedding"
quant_config = ModelOptNvFp4Config(exclude_modules=[prefix])
@@ -2289,162 +2279,6 @@ def _make_conv_metadata(
return metadata, token_offset
@pytest.mark.skipif(not torch.cuda.is_available(), reason="fused conv needs CUDA")
@pytest.mark.parametrize("state_layout", ["SD", "DS"])
@pytest.mark.parametrize(
"case",
[
pytest.param(
_ConvBatchCase(num_decodes=1, channels=512),
id="decode-single",
),
pytest.param(
_ConvBatchCase(
num_decodes=33,
channels=2048,
kernel_size=5,
dilation=1,
),
id="decode-padded",
),
pytest.param(
_ConvBatchCase(prefill_query_lens=(37, 0, 5, 128)),
id="prefill",
),
pytest.param(
_ConvBatchCase(
prefill_query_lens=(16, 676),
channels=10240,
state_index_stride=4,
include_null_state=False,
),
id="prefill-strided-state-indices",
),
pytest.param(
_ConvBatchCase(
spec_query_lens=(1, 4, 4, 0),
num_accepted=(1, 2, 4, 1),
spec_query_len=4,
graph_padding=4,
),
id="spec-graph-padded",
),
pytest.param(
_ConvBatchCase(
spec_query_lens=(3, 4),
num_accepted=(2, 4),
num_decodes=1,
prefill_query_lens=(5,),
spec_query_len=4,
),
id="mixed",
),
pytest.param(
_ConvBatchCase(
spec_query_lens=(1, 2),
num_accepted=(0, 1),
num_decodes=2,
prefill_query_lens=(0, 8, 3),
channels=640,
kernel_size=3,
dilation=2,
spec_query_len=3,
),
id="mixed-varied",
),
],
)
@pytest.mark.parametrize(
("layer_cls", "fuse_residual"),
[
pytest.param(Qwen4ExpPLELayer, True, id="nvidia-fused"),
pytest.param(Qwen4ExpPLELayer, False, id="nvidia-unfused"),
pytest.param(Qwen4ExpPLELayerAMD, True, id="amd-fused"),
],
)
def test_fused_conv_correctness(
case: _ConvBatchCase,
state_layout: str,
layer_cls: type[nn.Module],
fuse_residual: bool,
) -> None:
device = torch.device("cuda")
metadata, num_real_tokens = _make_conv_metadata(case, device)
if case.state_index_stride > 1:
assert metadata.state_indices_tensor.stride(0) == case.state_index_stride
# layer_cls is type[nn.Module], so layer_cls.__new__ is type.__new__.
module = object.__new__(layer_cls)
nn.Module.__init__(module)
module.conv_state_len = (case.kernel_size - 1) * case.dilation
module.short_conv_dilation = case.dilation
rng, state_reference, conv_state, weights = _make_conv_case(
device,
seed=num_real_tokens + case.channels,
channels=case.channels,
kernel_size=case.kernel_size,
dilation=case.dilation,
state_layout=state_layout,
spec_query_len=case.spec_query_len,
)
inputs = torch.randn(
metadata.num_actual_tokens,
case.channels,
device=device,
dtype=torch.bfloat16,
generator=rng,
)
residual = torch.randn(
inputs.shape,
device=device,
dtype=torch.bfloat16,
generator=rng,
)
outer_residual = torch.randn(
inputs.shape,
device=device,
dtype=torch.bfloat16,
generator=rng,
)
null_state = conv_state[NULL_BLOCK_ID].clone()
residual_kernel = residual.clone()
# The unfused kernel must overwrite the buffer without reading its contents.
residual_reference = (
residual.clone() if fuse_residual else torch.zeros_like(residual)
)
module._short_conv_dilated_dispatch(
inputs=inputs,
residual=residual_kernel,
outer_residual=outer_residual if fuse_residual else None,
metadata=metadata,
conv_state=conv_state,
conv_weights=weights,
)
_short_conv_dilated_dispatch_pytorch(
inputs=inputs,
residual=residual_reference,
metadata=metadata,
conv_state=state_reference,
conv_weights=weights,
conv_state_len=module.conv_state_len,
dilation=module.short_conv_dilation,
)
if fuse_residual:
residual_reference = outer_residual + residual_reference
assert torch.equal(
residual_kernel[:num_real_tokens], residual_reference[:num_real_tokens]
)
assert torch.equal(conv_state, state_reference)
assert torch.equal(conv_state[NULL_BLOCK_ID], null_state)
if case.graph_padding:
assert torch.equal(
residual_kernel[num_real_tokens:],
residual_reference[num_real_tokens:],
)
@pytest.mark.skipif(not torch.cuda.is_available(), reason="fused gate needs CUDA")
@pytest.mark.parametrize(("num_tokens", "strided_kv"), [(1, False), (64, True)])
def test_fused_gate_correctness(num_tokens: int, strided_kv: bool) -> None:
@@ -2509,273 +2343,3 @@ def test_fused_gate_correctness(num_tokens: int, strided_kv: bool) -> None:
expected_normed = grouped_norm(expected_gated, norm_conv)
assert torch.equal(gated, expected_gated)
torch.testing.assert_close(normed, expected_normed, atol=1e-2, rtol=1e-2)
def _build_amd_ngram_embedding(
monkeypatch: pytest.MonkeyPatch,
*,
device: str,
cpu_offload: bool,
fp8_checkpoint: bool,
) -> tuple[amd_ple_layer.Qwen4ExpNGramEmbedding, torch.Tensor]:
"""Construct and load a small AMD embedding through its checkpoint loader."""
_mock_etp_group(monkeypatch)
monkeypatch.setattr(embedding_module, "get_tensor_model_parallel_rank", lambda: 0)
monkeypatch.setattr(
embedding_module, "get_tensor_model_parallel_world_size", lambda: 1
)
monkeypatch.setattr(parameter_module, "get_tensor_model_parallel_rank", lambda: 0)
monkeypatch.setattr(
parameter_module, "get_tensor_model_parallel_world_size", lambda: 1
)
monkeypatch.setattr(
amd_ple_layer,
"get_current_vllm_config",
lambda: SimpleNamespace(engram_config=SimpleNamespace(cpu_offload=cpu_offload)),
)
config = Qwen4ExpTextConfig(
ngram_size=3,
heads_per_ngram=1,
ngram_vocab_size_base=5,
make_ngram_vocab_size_divisible_by=8,
split_ngram_parts=2,
ple_embed_dim=6,
eos_token_id=0,
vocab_size=64,
)
with torch.device(device):
module = amd_ple_layer.Qwen4ExpNGramEmbedding(
config,
embedding_dim=config.ple_embed_dim,
ple_dense_layer_id=0,
max_total_tokens=4,
prefix="test.ple_embedding",
layer_name="test.ple",
quant_config=(
Fp8Config(is_checkpoint_fp8_serialized=True) if fp8_checkpoint else None
),
params_dtype=torch.bfloat16,
)
# Only the outer owner is a stub; the embedding constructor and loader run.
layer = Qwen4ExpPLELayerAMD.__new__(Qwen4ExpPLELayerAMD)
nn.Module.__init__(layer)
layer.ple_embedding = module
monkeypatch.setattr(
amd_ple_layer,
"get_forward_context",
lambda: SimpleNamespace(no_compile_layers={module.layer_name: layer}),
)
embedding = module.ngram_embedding
loaded_weight = (
torch.arange(embedding.org_vocab_size * embedding.embedding_dim)
.reshape(embedding.org_vocab_size, embedding.embedding_dim)
.to(embedding.weight.dtype)
)
shard_size = (embedding.org_vocab_size + 1) // 2
weights = [
(f"ngram_embedding.shard_{i}.weight", shard)
for i, shard in enumerate(loaded_weight.split(shard_size))
]
expected_loaded = {"ngram_embedding.weight"}
if fp8_checkpoint:
weights.append(("ngram_embedding.weight_scale", torch.tensor([0.25])))
expected_loaded.add("ngram_embedding.weight_scale")
assert module.load_weights(weights) == expected_loaded
embedding.quant_method.process_weights_after_loading(embedding)
return module, loaded_weight
@pytest.mark.parametrize(
"device,cpu_offload",
[
pytest.param("cpu", False, id="cpu-resident"),
pytest.param(
"cuda:0",
False,
id="gpu-resident",
marks=pytest.mark.skipif(
not torch.cuda.is_available(), reason="requires CUDA or ROCm"
),
),
pytest.param(
"cuda:0",
True,
id="gpu-offloaded",
marks=pytest.mark.skipif(
not torch.cuda.is_available(), reason="requires CUDA or ROCm"
),
),
],
)
def test_amd_fp8_embedding_loads_checkpoint_shards_and_global_scale(
monkeypatch: pytest.MonkeyPatch,
device: str,
cpu_offload: bool,
) -> None:
"""AMD checkpoint loading must accept and apply the global FP8 scale."""
module, loaded_weight = _build_amd_ngram_embedding(
monkeypatch, device=device, cpu_offload=cpu_offload, fp8_checkpoint=True
)
embedding = module.ngram_embedding
assert embedding.weight.dtype == torch.float8_e4m3fn
assert embedding.weight.device == torch.device("cpu" if cpu_offload else device)
if cpu_offload:
assert embedding.weight.is_pinned()
assert embedding.weight_scale.device == torch.device(device)
torch.testing.assert_close(embedding.weight.float().cpu(), loaded_weight.float())
dequantized = embedding.dequantize(embedding.weight.to(device), torch.bfloat16)
torch.testing.assert_close(
dequantized.cpu(), loaded_weight.to(torch.bfloat16) * 0.25, rtol=0, atol=0
)
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA or ROCm")
def test_amd_pinned_embedding_prefetch_machinery_is_lazy(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""The pinned embedding must not allocate the prefetch stream/buffer in init.
Paths that never prefetch (AMD uses the synchronous pinned-lookup op)
keep the stream/buffer unallocated for the process lifetime. NVIDIA's
first ``start_prefetch`` — the eager profile run, always before cudagraph
capture — allocates them lazily.
"""
module, _ = _build_amd_ngram_embedding(
monkeypatch, device="cuda:0", cpu_offload=True, fp8_checkpoint=False
)
embedding = module.ngram_embedding
assert embedding._prefetch_buffer is None
assert embedding._prefetch_stream is None
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA or ROCm")
@pytest.mark.parametrize("fp8_checkpoint", [False, True], ids=["bf16", "fp8"])
def test_amd_pinned_embedding_output_written_under_compile(
monkeypatch: pytest.MonkeyPatch,
fp8_checkpoint: bool,
) -> None:
"""The AMD pinned lookup op must survive aot_autograd/Inductor.
The op writes via the declared-mutated ``output``. Without
``mutates_args=["output"]`` the call is dead-code-eliminated, the UVA lookup
never runs, and the returned ``output`` stays untouched.
"""
module, loaded_weight = _build_amd_ngram_embedding(
monkeypatch, device="cuda:0", cpu_offload=True, fp8_checkpoint=fp8_checkpoint
)
ngram_ids = torch.tensor([[3, 0], [1, 2]], device="cuda:0")
output = torch.zeros(
ngram_ids.shape[0],
module.embedding_dim,
dtype=module.ngram_embedding.weight.dtype,
device="cuda:0",
)
def lookup(ids: torch.Tensor, out: torch.Tensor) -> torch.Tensor:
torch.ops.vllm.qwen4_exp_amd_ple_ngram_embedding_pinned(
ids, out, module.layer_name
)
return out
torch.compile(lookup, fullgraph=True)(ngram_ids, output)
torch.cuda.current_stream().synchronize()
expected = loaded_weight[ngram_ids.cpu()].to(device="cuda:0").flatten(-2)
torch.testing.assert_close(output.float(), expected.float(), rtol=0, atol=0)
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA or ROCm")
@pytest.mark.parametrize("fp8_checkpoint", [False, True], ids=["bf16", "fp8"])
def test_amd_pinned_embedding_output_written_under_cudagraph_capture(
monkeypatch: pytest.MonkeyPatch,
fp8_checkpoint: bool,
) -> None:
"""The AMD pinned lookup op must be capture-safe.
It launches a single kernel on the current stream, so a cudagraph capture
reproduces the lookup into ``output`` without cross-stream work.
"""
module, loaded_weight = _build_amd_ngram_embedding(
monkeypatch, device="cuda:0", cpu_offload=True, fp8_checkpoint=fp8_checkpoint
)
ngram_ids = torch.tensor([[3, 0], [1, 2]], device="cuda:0")
output = torch.zeros(
ngram_ids.shape[0],
module.embedding_dim,
dtype=module.ngram_embedding.weight.dtype,
device="cuda:0",
)
def lookup(ids: torch.Tensor, out: torch.Tensor) -> torch.Tensor:
torch.ops.vllm.qwen4_exp_amd_ple_ngram_embedding_pinned(
ids, out, module.layer_name
)
return out
graph = torch.cuda.CUDAGraph()
warmup_stream = torch.cuda.Stream(device="cuda:0")
warmup_stream.wait_stream(torch.cuda.current_stream())
with torch.cuda.stream(warmup_stream):
for _ in range(3):
lookup(ngram_ids, output)
torch.cuda.current_stream().wait_stream(warmup_stream)
with torch.cuda.graph(graph):
lookup(ngram_ids, output)
for _ in range(2):
ngram_ids.copy_(ngram_ids.flip(-1) + 3)
output.fill_(float("nan"))
graph.replay()
torch.cuda.current_stream().synchronize()
expected = loaded_weight[ngram_ids.cpu()].to(device="cuda:0").flatten(-2)
torch.testing.assert_close(output.float(), expected.float(), rtol=0, atol=0)
@pytest.mark.skipif(
not current_platform.is_rocm(),
reason="only the ROCm backend runs the n-gram embedding under torch.compile",
)
@pytest.mark.parametrize(
("query_lens", "contexts"),
[([3, 1], [[5, 6], [0, 7]]), ([1, 0, 2], [[1, 2], [3, 4], [0, 0]])],
ids=["two-requests", "empty-request"],
)
def test_amd_ngram_embedding_matches_reference_under_compile(
monkeypatch: pytest.MonkeyPatch,
query_lens: list[int],
contexts: list[list[int]],
) -> None:
"""The AMD n-gram ID op must run under Dynamo with a symbolic request count."""
module, loaded_weight = _build_amd_ngram_embedding(
monkeypatch, device="cuda:0", cpu_offload=False, fp8_checkpoint=False
)
device = torch.device("cuda:0")
query_start_loc = torch.tensor(
[0, *accumulate(query_lens)], dtype=torch.int32, device=device
)
input_ids = torch.tensor(
[9, 0, 11, 12][: sum(query_lens)], dtype=torch.int32, device=device
)
ngram_context = torch.tensor(contexts, dtype=torch.int32, device=device)
compiled = torch.compile(module, fullgraph=True, dynamic=True)
output = compiled(input_ids, query_start_loc, ngram_context)
ngram_ids = _reference_ngram_ids(
input_ids,
query_start_loc,
ngram_context,
module.layer_multipliers,
module.ngram_heads_vocab_sizes,
module.ngram_heads_offsets,
module.eos_token_id,
module.heads_per_ngram,
)
expected = loaded_weight[ngram_ids.cpu()].flatten(-2).to(device)
torch.testing.assert_close(output.float(), expected.float(), rtol=0, atol=0)
File diff suppressed because it is too large Load Diff
@@ -1,181 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Unit tests for the Transformers modeling backend's MLA fuser.
The fuser must discover every MLA submodule structurally (never by assuming the
Transformers attribute names) and, when the query is low-rank, merge the checkpoint's
separate `q_a_proj`/`kv_a_proj_with_mqa` weights into the single fused down-projection.
"""
import pytest
import torch
import torch.nn as nn
import torch.nn.functional as F
from vllm.model_executor.models.transformers.fuser import get_fuser
from vllm.model_executor.models.transformers.fusers import MLAFuser
from vllm.model_executor.models.transformers.fx_utils import trace
_FUSED_QKV_A_PROJ = MLAFuser.merged_name
def _attention(q_lora_rank: int | None):
"""A meta `DeepseekV2Attention` in the requested q_lora configuration."""
pytest.importorskip("transformers.models.deepseek_v2.modeling_deepseek_v2")
from transformers.models.deepseek_v2.configuration_deepseek_v2 import (
DeepseekV2Config,
)
from transformers.models.deepseek_v2.modeling_deepseek_v2 import (
DeepseekV2Attention,
)
cfg = DeepseekV2Config(
hidden_size=256,
num_attention_heads=16,
kv_lora_rank=128,
qk_rope_head_dim=32,
qk_nope_head_dim=32,
v_head_dim=32,
q_lora_rank=q_lora_rank,
num_hidden_layers=1,
)
with torch.device("meta"):
return DeepseekV2Attention(cfg, layer_idx=0)
def _match(q_lora_rank: int | None) -> MLAFuser | None:
"""Match a meta `DeepseekV2Attention` directly (bypassing the per-class
`get_fuser` cache, so both q_lora variants of the same class are seen)."""
attn = _attention(q_lora_rank)
return MLAFuser.match(trace(attn), attn)
def test_discovers_modules_without_q_lora():
fuser = _match(q_lora_rank=None)
assert isinstance(fuser, MLAFuser)
assert fuser.q_proj_name == "q_proj"
assert fuser.kv_a_proj_name == "kv_a_proj_with_mqa"
assert fuser.kv_a_layernorm_name == "kv_a_layernorm"
assert fuser.kv_b_proj_name == "kv_b_proj"
assert fuser.o_proj_name == "o_proj"
assert fuser.q_a_proj_name is None
# Nothing is stacked without a query LoRA.
assert fuser.packed_modules_mapping == {}
assert fuser.orig_to_new_stacked("model.layers.0.self_attn") == {}
def test_discovers_modules_with_q_lora():
fuser = _match(q_lora_rank=64)
assert isinstance(fuser, MLAFuser)
assert fuser.q_a_proj_name == "q_a_proj"
assert fuser.q_a_layernorm_name == "q_a_layernorm"
assert fuser.q_b_proj_name == "q_b_proj"
assert fuser.kv_a_proj_name == "kv_a_proj_with_mqa"
assert fuser.kv_a_layernorm_name == "kv_a_layernorm"
assert fuser.kv_b_proj_name == "kv_b_proj"
assert fuser.o_proj_name == "o_proj"
assert fuser.q_proj_name is None
def test_q_lora_stacks_qkv_a_proj():
"""The MLA layer reads `q_a_proj` and `kv_a_proj_with_mqa` fused into one
down-projection, so both checkpoint weights must remap into it."""
fuser = _match(q_lora_rank=64)
assert isinstance(fuser, MLAFuser)
prefix = "model.layers.0.self_attn"
merged = f"{prefix}.{_FUSED_QKV_A_PROJ}"
assert fuser.packed_modules_mapping == {
_FUSED_QKV_A_PROJ: ["q_a_proj", "kv_a_proj_with_mqa"]
}
assert fuser.orig_to_new_stacked(prefix) == {
f"{prefix}.q_a_proj": (merged, 0),
f"{prefix}.kv_a_proj_with_mqa": (merged, 1),
}
@pytest.mark.parametrize("q_lora_rank", [64, None])
def test_update_forward_rewrites_real_attention(q_lora_rank):
"""The source rewrite must survive the input-stability check.
The q-LoRA path hoists the merged down-projection to a top-level statement
while its calls sit inside the `q_lora_rank` branches, so the region it
spans is wide; a check that over-approximates aliasing would silently refuse
here and cost the fusion rather than fail loudly."""
fuser = _match(q_lora_rank)
assert isinstance(fuser, MLAFuser)
fuser.update_forward(_attention(q_lora_rank))
names = set(fuser.fused_forward.__code__.co_names)
if fuser.q_a_proj_name is not None:
assert _FUSED_QKV_A_PROJ in names
assert not {"q_a_proj", "kv_a_proj_with_mqa"} & names
class _Norm(nn.Module):
"""A real RMSNorm computation: `match` verifies chain norms via `RMSNormFuser`."""
def __init__(self, n: int):
super().__init__()
self.weight = nn.Parameter(torch.ones(n))
def forward(self, x):
variance = x.pow(2).mean(-1, keepdim=True)
return self.weight * (x * torch.rsqrt(variance + 1e-6))
class RenamedMLA(nn.Module):
"""An MLA-shaped attention whose children have non-standard names, proving
discovery is by structure and not attribute name."""
def __init__(self):
super().__init__()
heads, kv_lora, rope, nope, v, hidden = 4, 32, 8, 8, 16, 64
self.alpha = nn.Linear(hidden, heads * (nope + rope)) # q_proj
self.beta = nn.Linear(hidden, kv_lora + rope) # kv_a_proj_with_mqa
self.gamma = _Norm(kv_lora) # kv_a_layernorm
self.delta = nn.Linear(kv_lora, heads * (nope + v)) # kv_b_proj
self.omega = nn.Linear(heads * v, hidden) # o_proj (uncalled)
self.kv_lora, self.rope = kv_lora, rope
def forward(self, hidden_states):
q = self.alpha(hidden_states)
kv_lora, k_pe = torch.split(
self.beta(hidden_states), [self.kv_lora, self.rope], dim=-1
)
expanded = self.delta(self.gamma(kv_lora))
# Stand-in for the attention interface; `match` finds `o_proj` (omega) from
# the source as the Linear producing the returned value.
attn_output = expanded.sum() + q.sum() + k_pe.sum()
attn_output = self.omega(attn_output)
return attn_output
def test_discovers_modules_under_arbitrary_names():
"""Discovery is purely structural: `RenamedMLA` gives its children non-standard
names, and `match` still locates each projection by dataflow."""
with torch.device("meta"):
module = RenamedMLA()
fuser = MLAFuser.match(trace(module), module)
assert isinstance(fuser, MLAFuser)
assert fuser.q_a_proj_name is None
assert fuser.q_proj_name == "alpha"
assert fuser.kv_a_proj_name == "beta"
assert fuser.kv_a_layernorm_name == "gamma"
assert fuser.kv_b_proj_name == "delta"
assert fuser.o_proj_name == "omega"
class GLU(nn.Module):
"""A gated MLP: matches the GLU fuser, never MLA."""
def __init__(self):
super().__init__()
self.gate = nn.Linear(16, 16)
self.up = nn.Linear(16, 16)
def forward(self, x):
return self.up(F.silu(self.gate(x)))
def test_non_mla_is_not_matched():
with torch.device("meta"):
assert get_fuser(GLU(), MLAFuser) is None
@@ -1,611 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Unit tests for the Transformers modeling backend's MoE fuser."""
import pytest
import torch
import torch.nn as nn
import torch.nn.functional as F
from vllm.model_executor.models.transformers.fusers import MoEBlockFuser
from .test_linear import GLUMLP
class TopKRouter(nn.Module):
"""HF v5 top-k router: `linear -> softmax -> topk (-> renorm)`."""
def __init__(self, num_experts=8, hidden=16, top_k=2, sigmoid=False):
super().__init__()
self.top_k = top_k
self.sigmoid = sigmoid
self.weight = nn.Parameter(torch.zeros(num_experts, hidden))
def forward(self, hidden_states):
logits = F.linear(hidden_states, self.weight)
scores = torch.sigmoid(logits) if self.sigmoid else F.softmax(logits, dim=-1)
value, index = torch.topk(scores, self.top_k, dim=-1)
value = value / value.sum(dim=-1, keepdim=True)
return logits, value, index
class SelectionScoredRouter(TopKRouter):
"""Scores the selected logits only (GraniteMoE/Aria style), in fp32."""
def forward(self, hidden_states):
logits = F.linear(hidden_states, self.weight).float()
value, index = torch.topk(logits, self.top_k, dim=-1)
value = (
torch.sigmoid(value) if self.sigmoid else F.softmax(value, dim=-1)
).type_as(hidden_states)
return logits, value, index
class EpsilonRenormRouter(TopKRouter):
"""Renormalizes via an epsilon-offset denominator (DeepSeek-V3 style)."""
def forward(self, hidden_states):
logits = F.linear(hidden_states, self.weight)
scores = torch.sigmoid(logits)
value, index = torch.topk(scores, self.top_k, dim=-1)
denominator = value.sum(dim=-1, keepdim=True) + 1e-20
value /= denominator
return logits, value, index
class UntraceableRouter(TopKRouter):
"""Scores and selects, then loses the trace before the renormalization."""
def forward(self, hidden_states):
logits = F.linear(hidden_states, self.weight)
scores = F.softmax(logits, dim=-1)
value, index = torch.topk(scores, self.top_k, dim=-1)
if value.sum() > 0: # data-dependent branch: tracing stops here
value = value / value.sum(dim=-1, keepdim=True)
return logits, value, index
class ScaledRouter(TopKRouter):
"""Greedy router scaling its top-k weights (DeepSeek `routed_scaling_factor`)."""
def forward(self, hidden_states):
logits = F.linear(hidden_states, self.weight)
scores = F.softmax(logits, dim=-1)
value, index = torch.topk(scores, self.top_k, dim=-1)
value = value * 16.0
return logits, value, index
class Fp32Router(TopKRouter):
"""Router that computes its logits in fp32 (DeepSeek/GLM style)."""
def forward(self, hidden_states):
logits = F.linear(
hidden_states.type(torch.float32), self.weight.type(torch.float32)
)
scores = F.softmax(logits, dim=-1)
value, index = torch.topk(scores, self.top_k, dim=-1)
return logits, value.to(hidden_states.dtype), index
class GroupedRouter(TopKRouter):
"""Group-limited router (DeepSeek `group_limited_greedy`), scaled weights."""
def forward(self, hidden_states):
logits = F.linear(hidden_states, self.weight)
scores = F.softmax(logits, dim=-1)
group_scores = scores.view(-1, 4, 2).max(dim=-1).values
group_idx = torch.topk(group_scores, k=2, dim=-1)[1]
group_mask = torch.zeros_like(group_scores)
group_mask.scatter_(1, group_idx, 1)
score_mask = group_mask.unsqueeze(-1).expand(-1, 4, 2).reshape(-1, 8)
scores = scores.masked_fill(~score_mask.bool(), 0.0)
value, index = torch.topk(scores, self.top_k, dim=-1)
value = value * 16.0
return logits, value, index
class CorrectionRouter(nn.Module):
"""Router with a score-correction bias buffer (DeepSeek-V3 noaux) -> matched."""
def __init__(self, num_experts=8, hidden=16):
super().__init__()
self.weight = nn.Parameter(torch.zeros(num_experts, hidden))
self.register_buffer("e_score_correction_bias", torch.zeros(num_experts))
def forward(self, hidden_states):
logits = F.linear(hidden_states, self.weight)
scores = torch.sigmoid(logits) + self.e_score_correction_bias
_, index = torch.topk(scores, 2, dim=-1)
return logits, scores, index
class BiasedRouter(TopKRouter):
"""A valid top-k router but not `weight`-only (extra `bias` param) -> declined."""
def __init__(self):
super().__init__()
self.bias = nn.Parameter(torch.zeros(8))
def forward(self, hidden_states):
logits = F.linear(hidden_states, self.weight) + self.bias
scores = F.softmax(logits, dim=-1)
value, index = torch.topk(scores, self.top_k, dim=-1)
return logits, value, index
class DisconnectedRouter(TopKRouter):
"""linear+softmax+top-k present but top-k ignores the logits -> not a router."""
def forward(self, hidden_states):
logits = F.linear(hidden_states, self.weight)
_ = F.softmax(logits, dim=-1) # scored, but not consumed by top-k
value, index = torch.topk(hidden_states, self.top_k, dim=-1)
return logits, value, index
class ExtraInputRouter(TopKRouter):
"""Router taking a block-owned correction bias as a second input (HYV3)."""
def forward(self, hidden_states, bias=0.0):
logits = F.linear(hidden_states, self.weight)
_, index = torch.topk(logits.sigmoid() + bias, self.top_k, dim=-1)
value = logits.sigmoid().gather(1, index)
return logits, value, index
class MoEExperts(nn.Module):
"""Packed experts (3D weights); only its name (`experts`) matters here."""
def __init__(self, num_experts=8, hidden=16, inter=32):
super().__init__()
self.gate_up_proj = nn.Parameter(torch.zeros(num_experts, 2 * inter, hidden))
self.down_proj = nn.Parameter(torch.zeros(num_experts, hidden, inter))
def forward(self, hidden_states, index, weights):
return hidden_states
class MoEBlock(nn.Module):
"""Single-tensor MoE block (Qwen3-style); subclasses override `_shared`."""
def __init__(self, router_cls=TopKRouter):
super().__init__()
self.experts = MoEExperts()
self.gate = router_cls()
def _shared(self, x, logits):
"""The term added to the experts' output (none for a plain block)."""
return None
def forward(self, hidden_states):
x = hidden_states.reshape(-1, hidden_states.shape[-1])
logits, weights, index = self.gate(x)
out = self.experts(x, index, weights)
if (shared := self._shared(x, logits)) is not None:
out = out + shared
return out.reshape(hidden_states.shape)
class MoEBlockNoShared(MoEBlock):
"""No shared-expert child but a gate-derived add -> declined, not misread."""
def _shared(self, x, logits):
return logits.sum()
class MoEBlockShared(MoEBlock):
"""A block with a shared expert and its sigmoid gate (Qwen2-style)."""
def __init__(self):
super().__init__()
self.shared_expert = GLUMLP()
self.shared_expert_gate = nn.Linear(16, 1, bias=False)
def _shared(self, x, logits):
return torch.sigmoid(self.shared_expert_gate(x)) * self.shared_expert(x)
class MoEBlockSharedNoGate(MoEBlock):
"""A block with an ungated shared expert -> native, shared passed through."""
def __init__(self):
super().__init__()
self.shared_expert = GLUMLP()
def _shared(self, x, logits):
return self.shared_expert(x)
class MoEBlockTuple(MoEBlock):
"""A tuple-returning block (gpt-oss-style) -> must decline."""
def forward(self, hidden_states):
x = hidden_states.reshape(-1, hidden_states.shape[-1])
_, weights, index = self.gate(x)
return self.experts(x, index, weights), index
class MoEBlockTupleVar(MoEBlock):
"""Returns a name bound to a tuple, not a literal tuple -> must still decline."""
def forward(self, hidden_states):
x = hidden_states.reshape(-1, hidden_states.shape[-1])
_, weights, index = self.gate(x)
result = self.experts(x, index, weights), index
return result
class MoEBlockNestedTupleReturn(MoEBlock):
"""Tuple `return` in a nested helper; block returns one tensor -> still fuses."""
def forward(self, hidden_states):
def keep(a, b):
return a, b
x = hidden_states.reshape(-1, hidden_states.shape[-1])
_, weights, index = self.gate(x)
out, _ = keep(self.experts(x, index, weights), index)
return out.reshape(hidden_states.shape)
class PlainMLP(nn.Module):
"""A non-GLU FFN: `down(act(up(x)))`, no gating multiply."""
def __init__(self, hidden: int = 16, inter: int = 32):
super().__init__()
self.up_proj = nn.Linear(hidden, inter, bias=False)
self.down_proj = nn.Linear(inter, hidden, bias=False)
self.act_fn = nn.SiLU()
def forward(self, x):
return self.down_proj(self.act_fn(self.up_proj(x)))
class MoEBlockSharedNonGLU(MoEBlock):
"""A non-GLU shared expert -> detected by dataflow (no gate/up merge)."""
def __init__(self):
super().__init__()
self.shared_expert = PlainMLP()
def _shared(self, x, logits):
return self.shared_expert(x)
class MoEBlockUnaccounted(MoEBlock):
"""A weight-bearing child outside the fused dataflow (pre-router) -> declined."""
def __init__(self):
super().__init__()
self.extra = nn.Linear(16, 16, bias=False)
def forward(self, hidden_states):
x = self.extra(hidden_states.reshape(-1, hidden_states.shape[-1]))
_, weights, index = self.gate(x)
return self.experts(x, index, weights).reshape(hidden_states.shape)
class BufferScale(nn.Module):
"""A stateful child carrying only a buffer (no parameters)."""
def __init__(self, hidden: int = 16):
super().__init__()
self.register_buffer("scale", torch.ones(hidden))
def forward(self, x):
return x * self.scale
class MoEBlockUnaccountedBuffer(MoEBlockUnaccounted):
"""Like `MoEBlockUnaccounted`, but the extra child holds only a buffer."""
def __init__(self):
super().__init__()
self.extra = BufferScale()
class MoEBlockExpertsKeywords(MoEBlock):
"""Passes the experts' routing by keyword -> still fuses."""
def forward(self, hidden_states):
x = hidden_states.reshape(-1, hidden_states.shape[-1])
_, weights, index = self.gate(x)
return self.experts(x, weights=weights, index=index)
class ScaledExperts(MoEExperts):
def forward(self, hidden_states, index, weights, scale=1.0):
return hidden_states * scale
class MoEBlockExpertsExtraArg(MoEBlock):
"""Passes the experts an extra argument the fused forward would drop."""
def __init__(self):
super().__init__()
self.experts = ScaledExperts()
def forward(self, hidden_states):
x = hidden_states.reshape(-1, hidden_states.shape[-1])
_, weights, index = self.gate(x)
return self.experts(x, index, weights, scale=2.0)
class MoEBlockSharedUnflattened(MoEBlockSharedNoGate):
"""Gate and shared expert read the unflattened input (Kolibri-1)."""
def forward(self, hidden_states):
_, weights, index = self.gate(hidden_states)
x = hidden_states.view(-1, hidden_states.shape[-1])
out = self.experts(x, index, weights).view(*hidden_states.shape)
return out + self.shared_expert(hidden_states)
class MoEBlockSharedAveraged(MoEBlockSharedUnflattened):
"""Averages the experts' and shared expert's outputs (Cohere2-MoE)."""
def forward(self, hidden_states):
return super().forward(hidden_states) / 2
class MoEBlockScaled(MoEBlockSharedNoGate):
"""Scales the experts' output by `routed_scaling_factor` (Laguna)."""
def forward(self, hidden_states):
x = hidden_states.reshape(-1, hidden_states.shape[-1])
_, weights, index = self.gate(x)
return self.experts(x, index, weights) * 2.5 + self.shared_expert(x)
class MoEBlockScaledBiasedRouter(MoEBlockScaled):
"""An unrecognized router with a scaled experts' output."""
def __init__(self):
super().__init__()
self.gate = BiasedRouter()
class MoEBlockSharedBiasedRouter(MoEBlockSharedUnflattened):
"""An unrecognized router with a shared expert."""
def __init__(self):
super().__init__()
self.gate = BiasedRouter()
class MoEBlockGateExtraInput(MoEBlock):
"""Passes the gate a block-owned bias the fused forward would drop (HYV3)."""
def __init__(self):
super().__init__(ExtraInputRouter)
self.register_buffer("e_score_correction_bias", torch.zeros(8))
def forward(self, hidden_states):
x = hidden_states.reshape(-1, hidden_states.shape[-1])
_, weights, index = self.gate(x, self.e_score_correction_bias)
return self.experts(x, index, weights)
class MoEBlockCastCombine(MoEBlockSharedNoGate):
"""Combines the outputs in fp32 (HYV3)."""
def forward(self, hidden_states):
x = hidden_states.reshape(-1, hidden_states.shape[-1])
_, weights, index = self.gate(x)
out = self.experts(x, index, weights).float() + self.shared_expert(x).float()
return out.to(x.dtype)
class MoEBlockIdentityLatent(MoEBlockSharedNoGate):
"""Wraps the experts in unused (identity) latent projections (NemotronH)."""
def __init__(self):
super().__init__()
self.fc1_latent_proj = nn.Identity()
self.fc2_latent_proj = nn.Identity()
def forward(self, hidden_states):
x = hidden_states.reshape(-1, hidden_states.shape[-1])
_, weights, index = self.gate(x)
out = self.fc2_latent_proj(
self.experts(self.fc1_latent_proj(x), index, weights)
)
return out + self.shared_expert(x)
@pytest.mark.parametrize("sigmoid", [False, True])
def test_moe_fuser_detects_router(sigmoid):
with torch.device("meta"):
block = MoEBlock(lambda: TopKRouter(sigmoid=sigmoid))
fuser = MoEBlockFuser.match(block, "experts")
assert isinstance(fuser, MoEBlockFuser)
assert fuser.gate_name == "gate"
assert fuser.scoring_func == ("sigmoid" if sigmoid else "softmax")
assert fuser.shared_name is None and fuser.shared_gate_name is None
@pytest.mark.parametrize("sigmoid", [False, True])
def test_moe_fuser_matches_router_scoring_the_selection(sigmoid):
"""Scoring the selection rather than all logits still allows the fuser to match."""
with torch.device("meta"):
block = MoEBlock(lambda: SelectionScoredRouter(sigmoid=sigmoid))
fuser = MoEBlockFuser.match(block, "experts")
assert isinstance(fuser, MoEBlockFuser)
assert fuser.gate_name == "gate"
assert fuser.scoring_func == ("sigmoid" if sigmoid else "softmax")
assert fuser.renormalize is (not sigmoid)
assert fuser.router_dtype == torch.float32
def test_moe_fuser_matches_scaled_router():
"""Weight scaling after the top-k (DeepSeek style) does not break matching."""
with torch.device("meta"):
block = MoEBlock(ScaledRouter)
assert isinstance(MoEBlockFuser.match(block, "experts"), MoEBlockFuser)
@pytest.mark.parametrize(
"router_cls,expected",
[
(TopKRouter, True), # divides by the sum of the selection
(EpsilonRenormRouter, True), # ... with an epsilon in the denominator
(ScaledRouter, False), # scales the weights instead of renormalizing
(UntraceableRouter, None), # trace stops before it could say
],
)
def test_moe_fuser_reads_renormalize_from_the_gate(router_cls, expected):
"""The router's graph, not the config, decides whether vLLM renormalizes: it
is the code being replaced, and it already bakes in `norm_topk_prob` because
the flag is a Python bool that fx resolves at trace time. Absence of the
division only means "no" on a complete trace."""
with torch.device("meta"):
block = MoEBlock(router_cls)
fuser = MoEBlockFuser.match(block, "experts")
assert isinstance(fuser, MoEBlockFuser)
assert fuser.renormalize is expected
def test_moe_fuser_matches_grouped_router():
"""Group masking between the score and the routing top-k still matches: the
scorer is anchored on the routing (last) top-k, not the group one."""
with torch.device("meta"):
block = MoEBlock(GroupedRouter)
fuser = MoEBlockFuser.match(block, "experts")
assert isinstance(fuser, MoEBlockFuser)
assert fuser.scoring_func == "softmax"
def test_moe_fuser_matches_correction_router():
"""A score-correction bias buffer (DeepSeek-V3 noaux) is allowed; the
rebuilt gate carries it in fp32 for FusedMoE's biased routers."""
with torch.device("meta"):
block = MoEBlock(CorrectionRouter)
fuser = MoEBlockFuser.match(block, "experts")
assert isinstance(fuser, MoEBlockFuser)
assert fuser.scoring_func == "sigmoid"
def test_moe_fuser_reads_router_dtype_from_the_gate():
"""A router that computes its logits in fp32 must keep routing in fp32 when
rebuilt, even though no config field names the dtype. The cast back to the
activation dtype after the top-k must not be mistaken for the routing dtype."""
with torch.device("meta"):
fp32 = MoEBlockFuser.match(MoEBlock(Fp32Router), "experts")
default = MoEBlockFuser.match(MoEBlock(TopKRouter), "experts")
assert fp32.router_dtype == torch.float32
assert default.router_dtype is None
def test_moe_fuser_detects_shared_experts():
with torch.device("meta"):
block = MoEBlockShared()
fuser = MoEBlockFuser.match(block, "experts")
assert isinstance(fuser, MoEBlockFuser)
assert fuser.shared_name == "shared_expert"
assert fuser.shared_gate_name == "shared_expert_gate"
def test_moe_fuser_shared_without_gate():
with torch.device("meta"):
block = MoEBlockSharedNoGate()
fuser = MoEBlockFuser.match(block, "experts")
assert isinstance(fuser, MoEBlockFuser)
assert fuser.shared_name == "shared_expert"
assert fuser.shared_gate_name is None
def test_moe_fuser_detects_non_glu_shared_expert():
with torch.device("meta"):
block = MoEBlockSharedNonGLU()
fuser = MoEBlockFuser.match(block, "experts")
assert isinstance(fuser, MoEBlockFuser)
# Recognised by dataflow (added to the experts' output), though not a GLU.
assert fuser.shared_name == "shared_expert"
assert fuser.shared_gate_name is None
@pytest.mark.parametrize(
"block_cls",
[
lambda: MoEBlock(BiasedRouter), # router with an unrecognized extra param
MoEBlockTuple, # tuple-returning block (e.g. gpt-oss)
MoEBlockTupleVar, # tuple returned via a name binding, not a literal
MoEBlockUnaccounted, # weight-bearing child outside the fused dataflow
MoEBlockUnaccountedBuffer, # buffer-only child outside the fused dataflow
MoEBlockExpertsExtraArg, # extra experts argument the fused forward drops
MoEBlockNoShared, # gate-derived term added to the experts' output
MoEBlockSharedAveraged, # experts' and shared expert's outputs averaged
MoEBlockGateExtraInput, # gate input the fused forward drops
],
)
def test_moe_fuser_declines_unsupported(block_cls):
with torch.device("meta"):
block = block_cls()
assert MoEBlockFuser.match(block, "experts") is None
def test_moe_fuser_ignores_nested_returns():
"""A tuple `return` inside a nested helper must not decline a block whose own
forward returns a single tensor."""
with torch.device("meta"):
block = MoEBlockNestedTupleReturn()
assert isinstance(MoEBlockFuser.match(block, "experts"), MoEBlockFuser)
def test_moe_fuser_router_requires_connected_dataflow():
"""A gate with linear + softmax + top-k present but not wired as a router
(top-k selects over the input, not the scored logits) is not detected."""
with torch.device("meta"):
block = MoEBlock(DisconnectedRouter)
assert MoEBlockFuser.match(block, "experts") is None
def test_moe_fuser_unrecognized_router_moves_shared_expert():
"""Without vLLM routing, the shared expert still moves to vLLM."""
with torch.device("meta"):
block = MoEBlockSharedBiasedRouter()
fuser = MoEBlockFuser.match(block, "experts")
assert fuser.scoring_func is None
assert fuser.shared_name == "shared_expert"
@pytest.mark.parametrize(
("block_cls", "scoring_func"),
[
(MoEBlockSharedUnflattened, None), # routing falls back to Transformers
(MoEBlock, "softmax"), # no shared output to divide
],
)
def test_moe_fuser_divided_shared_output(block_cls, scoring_func):
with torch.device("meta"):
fuser = MoEBlockFuser.match(block_cls(), "experts", divides_shared_output=True)
assert fuser.scoring_func == scoring_func
def test_moe_fuser_scaled_output_needs_vllm_routing():
"""Only vLLM routing applies the routed scale, so nothing else moves without it."""
with torch.device("meta"):
block = MoEBlockScaledBiasedRouter()
assert MoEBlockFuser.match(block, "experts") is None
def test_moe_fuser_matches_experts_called_by_keyword():
with torch.device("meta"):
block = MoEBlockExpertsKeywords()
assert isinstance(MoEBlockFuser.match(block, "experts"), MoEBlockFuser)
def test_moe_fuser_scaled_output_with_vllm_routing():
"""A scaled block fuses because vLLM routing applies `routed_scaling_factor`."""
with torch.device("meta"):
fuser = MoEBlockFuser.match(MoEBlockScaled(), "experts")
assert fuser.scoring_func == "softmax"
assert fuser.shared_name == "shared_expert"
@pytest.mark.parametrize("block_cls", [MoEBlockCastCombine, MoEBlockIdentityLatent])
def test_moe_fuser_looks_through_casts_and_identity(block_cls):
with torch.device("meta"):
fuser = MoEBlockFuser.match(block_cls(), "experts")
assert fuser.shared_name == "shared_expert"
@@ -1,409 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Unit tests for the Transformers modeling backend's RMSNorm fuser."""
from types import SimpleNamespace
import pytest
import torch
import torch.nn as nn
import torch.nn.functional as F
from vllm.model_executor.models.transformers.fuser import get_fuser
from vllm.model_executor.models.transformers.fusers import RMSNormFuser
class RMSNorm(nn.Module):
"""The canonical HF RMSNorm: `weight * x * rsqrt(mean(x**2) + eps)`."""
def __init__(self, hidden: int = 16, eps: float = 1e-5, weight: bool = True):
super().__init__()
if weight:
self.weight = nn.Parameter(torch.ones(hidden))
self.variance_epsilon = eps
def _rms(self, x):
return x * torch.rsqrt(x.pow(2).mean(-1, keepdim=True) + self.variance_epsilon)
def forward(self, x):
return self.weight * self._rms(x.to(torch.float32)).to(x.dtype)
class GemmaRMSNorm(RMSNorm):
"""Zero-centered weight: `(1 + weight) * normalized`."""
def __init__(self, hidden: int = 16, eps: float = 1e-6):
super().__init__(hidden, eps)
self.weight = nn.Parameter(torch.zeros(hidden))
def forward(self, x):
return (1.0 + self.weight) * self._rms(x.to(torch.float32)).to(x.dtype)
class WeightlessRMSNorm(RMSNorm):
"""No scale parameter (e.g. Gemma3n `with_scale=False`)."""
def __init__(self, hidden: int = 16, eps: float = 1e-6):
super().__init__(hidden, eps, weight=False)
def forward(self, x):
return self._rms(x.to(torch.float32)).to(x.dtype)
class Gemma4RMSNorm(RMSNorm):
"""`pow(v, -0.5)` where the others write `rsqrt(v)` (HF `Gemma4RMSNorm._norm`).
Gemma 4 spells the reciprocal square root as `torch.pow(..., -0.5)` to keep
Torch and JAX in agreement, and -- unlike Gemma 1-3 -- its weight is *not*
zero-centered, so this fuses to a plain `RMSNorm`.
"""
def _rms(self, x):
mean_squared = x.pow(2).mean(-1, keepdim=True) + self.variance_epsilon
return x * torch.pow(mean_squared, -0.5)
def forward(self, x):
return (self._rms(x.float()) * self.weight.float()).type_as(x)
class PowOperatorRMSNorm(Gemma4RMSNorm):
"""The `v ** -0.5` operator spelling of the same reciprocal square root."""
def _rms(self, x):
return x * (x.pow(2).mean(-1, keepdim=True) + self.variance_epsilon) ** -0.5
class KwargPowRMSNorm(Gemma4RMSNorm):
"""The keyword spelling `torch.pow(v, exponent=-0.5)` of the same rsqrt."""
def _rms(self, x):
mean_squared = x.pow(2).mean(-1, keepdim=True) + self.variance_epsilon
return x * torch.pow(mean_squared, exponent=-0.5)
class KwargBasePowRMSNorm(Gemma4RMSNorm):
"""The fully functional spelling `torch.pow(input=v, exponent=-0.5)`."""
def _rms(self, x):
mean_squared = x.pow(2).mean(-1, keepdim=True) + self.variance_epsilon
return x * torch.pow(input=mean_squared, exponent=-0.5)
class WeightlessGemma4RMSNorm(Gemma4RMSNorm):
"""Gemma 4 with `with_scale=False`: the `pow` spelling and no scale parameter."""
def __init__(self, hidden: int = 16, eps: float = 1e-6):
super().__init__(hidden, eps, weight=False)
def forward(self, x):
return self._rms(x.float()).type_as(x)
class LayerNorm(RMSNorm):
"""An RMSNorm not named `*RMSNorm`, keeping the input dtype (no upcast)."""
def __init__(self, hidden: int = 16, eps: float = 1e-6):
super().__init__(hidden, eps)
def forward(self, x):
return self.weight * self._rms(x)
class NamedEpsRMSNorm(nn.Module):
"""Holds eps under `attr`, with no `variance_epsilon` to find instead."""
attr = "eps"
def __init__(self, hidden: int = 16, eps: float = 1e-6):
super().__init__()
self.weight = nn.Parameter(torch.ones(hidden))
setattr(self, self.attr, eps)
def forward(self, x):
eps = getattr(self, self.attr)
return self.weight * (x * torch.rsqrt(x.pow(2).mean(-1, True) + eps))
class ToleranceRMSNorm(NamedEpsRMSNorm):
"""Names eps something other than `eps`, so only its value identifies it."""
attr = "tolerance"
class LiteralEpsRMSNorm(RMSNorm):
"""eps is a literal in the source, and an unrelated attribute happens to
hold the same value: matching on the value alone would bind to it."""
def __init__(self, hidden: int = 16, eps: float = 1e-5):
super().__init__(hidden, eps)
self.epsilon_lookalike = 1e-4
def forward(self, x):
return self.weight * (x * torch.rsqrt(x.pow(2).mean(-1, True) + 1e-4))
class AmbiguousEpsRMSNorm(nn.Module):
"""Two attributes hold the eps value, and the forward reads the second, so
their order cannot pick it."""
def __init__(self, hidden: int = 16, eps: float = 1e-6):
super().__init__()
self.weight = nn.Parameter(torch.ones(hidden))
self.decoy_eps = eps
self.variance_epsilon = eps
def forward(self, x):
var = x.pow(2).mean(-1, True)
return self.weight * (x * torch.rsqrt(var + self.variance_epsilon))
class NotAnRMSNorm(RMSNorm):
"""Mean-subtracting LayerNorm-like math -> not an RMSNorm."""
def __init__(self, hidden: int = 16, eps: float = 1e-6):
super().__init__(hidden, eps)
def forward(self, x):
x = x - x.mean(-1, keepdim=True)
variance = x.var(-1, keepdim=True)
return self.weight * x / torch.sqrt(variance + self.variance_epsilon)
class GatedRMSNorm(RMSNorm):
"""Second input and tail compute -> not an RMSNorm."""
def forward(self, x, gate=None):
normed = self.weight * self._rms(x.to(torch.float32)).to(x.dtype)
return normed * F.silu(gate)
class GatedFusedRMSNorm(nn.Module):
"""Same as GatedRMSNorm, but built on the fused `rms_norm` op -> not an RMSNorm."""
def __init__(self, hidden: int = 16, eps: float = 1e-5):
super().__init__()
self.weight = nn.Parameter(torch.ones(hidden))
self.eps = eps
def forward(self, x, gate=None):
return F.rms_norm(x, (x.shape[-1],), self.weight, self.eps) * F.silu(gate)
class UntraceableGatedRMSNorm(RMSNorm):
"""Tracer can't see tail compute in forward, but still has a second input (gate)."""
def forward(self, x, gate=None):
normed = self.weight * self._rms(x.to(torch.float32)).to(x.dtype)
if gate.sum() > 0: # untraceable -> partial graph, no visible tail
normed = normed * F.silu(gate)
return normed
@pytest.mark.parametrize(
"cls,eps,zero_centered",
[
(RMSNorm, 1e-5, False),
(GemmaRMSNorm, 1e-6, True),
(WeightlessRMSNorm, 1e-6, False),
(Gemma4RMSNorm, 1e-6, False),
(PowOperatorRMSNorm, 1e-6, False),
(KwargPowRMSNorm, 1e-6, False),
(KwargBasePowRMSNorm, 1e-6, False),
(WeightlessGemma4RMSNorm, 1e-6, False),
(LayerNorm, 1e-6, False),
(torch.nn.RMSNorm, 1e-5, False), # fused `F.rms_norm` op
],
)
def test_detects_rms_norm_variants(cls, eps, zero_centered):
with torch.device("meta"):
fuser = get_fuser(cls(16, eps=eps), RMSNormFuser)
assert isinstance(fuser, RMSNormFuser)
assert fuser.zero_centered == zero_centered
@pytest.mark.parametrize("cls", [NotAnRMSNorm, nn.LayerNorm, nn.SiLU])
def test_non_rms_norms_are_not_matched(cls):
with torch.device("meta"):
module = cls(16) if cls is nn.LayerNorm else cls()
assert get_fuser(module, RMSNormFuser) is None
@pytest.mark.parametrize(
"cls", [GatedRMSNorm, GatedFusedRMSNorm, UntraceableGatedRMSNorm]
)
def test_gated_rms_norm_is_not_fused(cls):
with torch.device("meta"):
assert get_fuser(cls(), RMSNormFuser) is None
@pytest.mark.parametrize(
"cls,expected,zero_centered",
[
(RMSNorm, "RMSNorm", False),
(GemmaRMSNorm, "GemmaRMSNorm", True),
(WeightlessRMSNorm, "RMSNorm", False),
(Gemma4RMSNorm, "RMSNorm", False),
(KwargPowRMSNorm, "RMSNorm", False),
(KwargBasePowRMSNorm, "RMSNorm", False),
(WeightlessGemma4RMSNorm, "RMSNorm", False),
],
)
def test_rms_norm_builds_vllm_class(cls, expected, zero_centered, default_vllm_config):
from vllm.model_executor.layers.layernorm import GemmaRMSNorm as VLLMGemmaRMSNorm
from vllm.model_executor.layers.layernorm import RMSNorm as VLLMRMSNorm
with torch.device("meta"):
module = cls()
fuser = get_fuser(module, RMSNormFuser)
built = fuser.fuse(module, "norm", default_vllm_config)
from vllm.model_executor.models.transformers.fusers.rms_norm import (
TPAwareNormMixin,
)
types_by_name = {"RMSNorm": VLLMRMSNorm, "GemmaRMSNorm": VLLMGemmaRMSNorm}
assert isinstance(built, types_by_name[expected])
assert isinstance(built, TPAwareNormMixin)
assert built.variance_epsilon == module.variance_epsilon
# The weight states the hidden size (and is what the TP check reads); a norm
# without one has none to state.
weight = getattr(module, "weight", None)
assert isinstance(built.weight, nn.Parameter) == (weight is not None)
assert built.weight.shape[0] == (weight.size(0) if weight is not None else 0)
@pytest.mark.parametrize(
"cls", [Gemma4RMSNorm, PowOperatorRMSNorm, KwargPowRMSNorm, KwargBasePowRMSNorm]
)
def test_pow_spelled_norm_fuses_to_equivalent_math(cls, default_vllm_config):
"""Matching `pow(v, -0.5)` is only sound if it really is the reciprocal square
root: the fused norm must reproduce the unfused forward, not just replace it.
`pow` and `rsqrt` take different paths to the same value, so compare at float32
tolerance rather than bit-for-bit."""
torch.manual_seed(0)
module = cls(16)
with torch.no_grad():
module.weight.copy_(torch.randn(16))
built = get_fuser(module, RMSNormFuser).fuse(module, "norm", default_vllm_config)
with torch.no_grad():
built.weight.copy_(module.weight)
x = torch.randn(4, 16)
torch.testing.assert_close(built(x), module(x), rtol=1e-5, atol=1e-6)
def test_weightless_norm_has_no_hidden_size(default_vllm_config):
"""A weightless norm states no hidden size, and the model's (LM) hidden size
would be wrong for one on a sub-dimension: Llama 4's `qk_norm` normalizes
head_dim, so sizing it at hidden_size made the TP gather reject its input.
It fuses with none, and so normalizes the width it is given at any TP size,
matching the unfused norm and vLLM's native `RMSNorm(hidden_size=head_dim)`.
"""
module = WeightlessRMSNorm(72)
built = get_fuser(module, RMSNormFuser).fuse(module, "norm", default_vllm_config)
assert built.hidden_size == 0
built.tp_size = 2 # emulate TP=2 without a real process group
x = torch.randn(4, 72)
torch.testing.assert_close(built(x), module(x))
def test_fused_rms_norm_op_default_eps(default_vllm_config):
"""`torch.nn.RMSNorm` (a single `F.rms_norm` call) matches via the fast path;
its default `eps=None` resolves to `finfo(dtype).eps` in `fuse`."""
from vllm.model_executor.layers.layernorm import RMSNorm as VLLMRMSNorm
with torch.device("meta"):
module = torch.nn.RMSNorm(16) # forward is a single `F.rms_norm` call
fuser = get_fuser(module, RMSNormFuser)
assert isinstance(fuser, RMSNormFuser)
assert not fuser.zero_centered
vllm_config = SimpleNamespace(model_config=SimpleNamespace(dtype=torch.float32))
built = fuser.fuse(module, "norm", vllm_config)
assert isinstance(built, VLLMRMSNorm)
assert built.variance_epsilon == torch.finfo(torch.float32).eps
def test_eps_is_derived_per_instance(default_vllm_config):
"""Two instances of the same norm class with different eps must fuse to their
own eps: the type-cached fuser holds only structure, not this value."""
with torch.device("meta"):
for eps in (1e-5, 1e-6):
module = RMSNorm(16, eps=eps)
fuser = get_fuser(module, RMSNormFuser)
built = fuser.fuse(module, "norm", default_vllm_config)
assert built.variance_epsilon == eps
@pytest.mark.parametrize("cls", [NamedEpsRMSNorm, ToleranceRMSNorm])
def test_eps_attr_is_found_by_value_not_name(cls, default_vllm_config):
"""The eps attribute is identified by holding the traced value, so a norm
stays per-instance correct whatever it names it."""
with torch.device("meta"):
for eps in (1e-5, 1e-6):
module = cls(16, eps=eps)
fuser = get_fuser(module, RMSNormFuser)
assert fuser.eps_attr == cls.attr
built = fuser.fuse(module, "norm", default_vllm_config)
assert built.variance_epsilon == eps
def test_literal_eps_is_not_mistaken_for_an_attribute(default_vllm_config, caplog):
"""A literal eps is recognised as coming from no attribute, even when one
holds the same value, and is taken from the traced source instead."""
logger = "vllm.model_executor.models.transformers.fusers.rms_norm"
with caplog.at_level("DEBUG", logger=logger), torch.device("meta"):
module = LiteralEpsRMSNorm()
fuser = get_fuser(module, RMSNormFuser)
built = fuser.fuse(module, "norm", default_vllm_config)
assert fuser.eps_attr is None
assert built.variance_epsilon == 1e-4
assert "does not hold its eps" in caplog.text
def test_ambiguous_eps_attrs_are_disambiguated(default_vllm_config):
"""When several attributes hold the eps value, the one the forward actually
reads is identified, and they are all left as they were found."""
with torch.device("meta"):
module = AmbiguousEpsRMSNorm(16, eps=1e-6)
before = dict(vars(module))
fuser = get_fuser(module, RMSNormFuser)
assert fuser.eps_attr == "variance_epsilon"
assert vars(module) == before
other = AmbiguousEpsRMSNorm(16, eps=1e-5)
other.decoy_eps = 1e-3
built = fuser.fuse(other, "norm", default_vllm_config)
assert built.variance_epsilon == 1e-5
def test_fused_norm_is_gather_capable(default_vllm_config):
"""Every weighted fused norm is emitted gather-capable, so a norm on a head-sharded
projection (OLMoE-style) self-corrects at runtime with no QKV-specific
plumbing. A full-width input skips the gather and equals a plain norm."""
from vllm.model_executor.layers.layernorm import GemmaRMSNorm, RMSNorm
from vllm.model_executor.models.transformers.fusers import rms_norm
torch.manual_seed(0)
x = torch.randn(4, 16)
for gathered_cls, plain_cls in [
(rms_norm.TPAwareRMSNorm, RMSNorm),
(rms_norm.TPAwareGemmaRMSNorm, GemmaRMSNorm),
]:
gathered = gathered_cls(hidden_size=16, eps=1e-6)
assert isinstance(gathered, rms_norm.TPAwareNormMixin)
plain = plain_cls(hidden_size=16, eps=1e-6)
with torch.no_grad():
weight = torch.randn(16)
gathered.weight.copy_(weight)
plain.weight.copy_(weight)
torch.testing.assert_close(gathered(x), plain(x))
def test_gathered_norm_rejects_uneven_sharding(default_vllm_config):
"""A sharded input (narrower than the full-width weight) that does not tile
the weight evenly across ranks is rejected before any collective."""
from vllm.model_executor.models.transformers.fusers import rms_norm
norm = rms_norm.TPAwareRMSNorm(hidden_size=8, eps=1e-6)
norm.tp_size = 2 # emulate TP=2 without a real process group
with pytest.raises(ValueError, match="does not tile it evenly"):
norm(torch.randn(2, 3)) # 3 * 2 != 8
@@ -1,86 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for skipping dynamic rope's device sync where it can never rescale."""
from types import SimpleNamespace
import pytest
import torch
from transformers import HunYuanDenseV1Config, LlamaConfig
from transformers.models.hunyuan_v1_dense.modeling_hunyuan_v1_dense import (
HunYuanDenseV1RotaryEmbedding,
)
from transformers.models.llama.modeling_llama import LlamaRotaryEmbedding
from vllm.model_executor.models.transformers.utils import (
can_enable_torch_compile,
disable_noop_dynamic_rope,
)
MAX_POSITION_EMBEDDINGS = 128
CONFIG_KWARGS = dict(
hidden_size=64,
num_attention_heads=4,
num_hidden_layers=1,
max_position_embeddings=MAX_POSITION_EMBEDDINGS,
)
DYNAMIC = {"rope_type": "dynamic", "factor": 4.0, "rope_theta": 1e4}
def make_vllm_config(hf_config, max_model_len: int) -> SimpleNamespace:
model_config = SimpleNamespace(hf_config=hf_config, max_model_len=max_model_len)
return SimpleNamespace(model_config=model_config)
@pytest.mark.parametrize(
"rope_parameters,max_model_len,compilable",
[
(DYNAMIC, MAX_POSITION_EMBEDDINGS, True),
(DYNAMIC, MAX_POSITION_EMBEDDINGS + 1, False),
# The longrope layer type still syncs
(
{"full_attention": DYNAMIC, "sliding_attention": {"rope_type": "longrope"}},
MAX_POSITION_EMBEDDINGS,
False,
),
],
)
def test_can_enable_torch_compile(rope_parameters, max_model_len, compilable):
"""Dynamic rope only blocks compilation when it can still rescale."""
config = SimpleNamespace(
max_position_embeddings=MAX_POSITION_EMBEDDINGS,
rope_parameters=rope_parameters,
)
config.get_text_config = lambda: config
vllm_config = make_vllm_config(config, max_model_len)
assert can_enable_torch_compile(vllm_config) is compilable
@pytest.mark.parametrize(
"config,rotary_cls",
[
(LlamaConfig(**CONFIG_KWARGS, rope_parameters=DYNAMIC), LlamaRotaryEmbedding),
# Hunyuan's dynamic init also applies `alpha`, which must be kept
(
HunYuanDenseV1Config(
**CONFIG_KWARGS, head_dim=16, rope_parameters=DYNAMIC | {"alpha": 1e3}
),
HunYuanDenseV1RotaryEmbedding,
),
],
)
def test_disable_noop_dynamic_rope(monkeypatch, config, rotary_cls):
"""The update no longer syncs and the frequencies are unchanged."""
inputs = torch.zeros(1, 8, 64), torch.arange(8)[None, :]
rotary_emb = rotary_cls(config)
expected = rotary_emb(*inputs)
vllm_config = make_vllm_config(config, MAX_POSITION_EMBEDDINGS)
disable_noop_dynamic_rope(rotary_emb, vllm_config)
def no_sync(*args, **kwargs):
raise AssertionError("dynamic rope update still runs")
monkeypatch.setattr(torch, "max", no_sync)
actual = rotary_emb(*inputs)
assert all(torch.equal(e, a) for e, a in zip(expected, actual))
@@ -1,80 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Unit tests for the Transformers backend's hw-agnostic layer resolution.
`layers._resolve` imports a layer symbol from
`vllm.model_executor.hw_agnostic.layers.<module>` when `VLLM_USE_HW_AGNOSTIC`
is set and the symbol exists, and otherwise falls back to
`vllm.model_executor.layers.<module>`. These tests pin that contract and the
logging that reports which source was used.
CPU-only: no engine is built. The end-to-end counterpart that serves a model
through `vllm_runner` lives in `test_layer_registry_e2e.py` (GPU job).
"""
import logging
import sys
import types
import pytest
from vllm.model_executor.models.transformers import layers
HW_MODULE = "vllm.model_executor.hw_agnostic.layers.layernorm"
@pytest.fixture
def fake_hw_layernorm(monkeypatch):
"""Inject a hw-agnostic `layernorm` module exposing a sentinel `RMSNorm`.
A `SimpleNamespace` stands in for the module: `importlib.import_module`
returns it from `sys.modules` and `getattr` resolves `RMSNorm`, while its
attributes are set at construction (no `ModuleType` attribute-set that mypy
rejects, no constant `setattr` that ruff rejects)."""
module = types.SimpleNamespace(RMSNorm=type("HwRMSNorm", (), {}))
monkeypatch.setitem(sys.modules, HW_MODULE, module)
return module
def test_falls_back_to_vllm_when_disabled(monkeypatch, fake_hw_layernorm):
"""Disabled: the vLLM class is used even if a hw-agnostic one exists."""
monkeypatch.setenv("VLLM_USE_HW_AGNOSTIC", "0")
from vllm.model_executor.layers.layernorm import RMSNorm as VllmRMSNorm
assert layers._resolve("layernorm", "RMSNorm") is VllmRMSNorm
def test_uses_hw_agnostic_when_enabled(monkeypatch, fake_hw_layernorm, caplog_vllm):
"""Enabled and available: the hw-agnostic class is used and logged."""
monkeypatch.setenv("VLLM_USE_HW_AGNOSTIC", "1")
with caplog_vllm.at_level(logging.INFO):
resolved = layers._resolve("layernorm", "RMSNorm")
assert resolved is fake_hw_layernorm.RMSNorm
assert "Using hw-agnostic layer: RMSNorm" in caplog_vllm.text
def test_falls_back_when_symbol_missing(monkeypatch, caplog_vllm):
"""Enabled but the symbol is not ported: fall back to vLLM and warn."""
monkeypatch.setenv("VLLM_USE_HW_AGNOSTIC", "1")
# A hw-agnostic module without the requested attribute triggers fallback.
empty = types.ModuleType(HW_MODULE)
monkeypatch.setitem(sys.modules, HW_MODULE, empty)
from vllm.model_executor.layers.layernorm import RMSNorm as VllmRMSNorm
with caplog_vllm.at_level(logging.WARNING):
resolved = layers._resolve("layernorm", "RMSNorm")
assert resolved is VllmRMSNorm
assert "falling back to default" in caplog_vllm.text
def test_act_and_mul_falls_back_for_unknown_activation(
monkeypatch, default_vllm_config
):
"""An activation with no hw-agnostic equivalent falls back to vLLM's.
`default_vllm_config` supplies the config context the CustomOp needs.
"""
monkeypatch.setenv("VLLM_USE_HW_AGNOSTIC", "1")
from vllm.model_executor.layers.activation import GeluAndMul
assert isinstance(layers.get_act_and_mul_fn("gelu"), GeluAndMul)