[Mini-VLLM] Cut dead subsystems across serving, connectors, parsers

- Remove P/D transfer connectors: nixl, nixl_chat, mooncake x2, hf3fs,
  moriio, hisparse, example and hidden-states connectors
- Reduce weight_transfer to the base API used by model loading
- Remove kv_offload tiering, simple_kv_offload, ec cpu/example,
  snapshot command, engine snapshot server
- Remove transformers model fallback, dsv4 tokenizers, dead platforms
  cpu/tpu/xpu, flashmla third_party, dsa, megamoe/flydsl, dead attn
  ops, dead layers (short_conv, fused_embed_norm, indexer_topk,
  relu2 gemma_ar, lightning_attn, minimax_rms_norm), gemm_rs_ar
- Keep MoE LoRA op (imported eagerly by punica_gpu); drop fp8 variant
- Rebuild parser hubs around Qwen3 adapters only (36 dead adapters)
- Remove dead tests with the code they covered

Co-authored-by: Hinata <hinata@idihore.id>
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-10-19 02:19:03 +07:00
co-authored by Hinata
parent 54a396586e
commit 2b544b7ba0
353 changed files with 29 additions and 121269 deletions
File diff suppressed because it is too large Load Diff
@@ -1,181 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Unit tests for the Transformers modeling backend's MLA fuser.
The fuser must discover every MLA submodule structurally (never by assuming the
Transformers attribute names) and, when the query is low-rank, merge the checkpoint's
separate `q_a_proj`/`kv_a_proj_with_mqa` weights into the single fused down-projection.
"""
import pytest
import torch
import torch.nn as nn
import torch.nn.functional as F
from vllm.model_executor.models.transformers.fuser import get_fuser
from vllm.model_executor.models.transformers.fusers import MLAFuser
from vllm.model_executor.models.transformers.fx_utils import trace
_FUSED_QKV_A_PROJ = MLAFuser.merged_name
def _attention(q_lora_rank: int | None):
"""A meta `DeepseekV2Attention` in the requested q_lora configuration."""
pytest.importorskip("transformers.models.deepseek_v2.modeling_deepseek_v2")
from transformers.models.deepseek_v2.configuration_deepseek_v2 import (
DeepseekV2Config,
)
from transformers.models.deepseek_v2.modeling_deepseek_v2 import (
DeepseekV2Attention,
)
cfg = DeepseekV2Config(
hidden_size=256,
num_attention_heads=16,
kv_lora_rank=128,
qk_rope_head_dim=32,
qk_nope_head_dim=32,
v_head_dim=32,
q_lora_rank=q_lora_rank,
num_hidden_layers=1,
)
with torch.device("meta"):
return DeepseekV2Attention(cfg, layer_idx=0)
def _match(q_lora_rank: int | None) -> MLAFuser | None:
"""Match a meta `DeepseekV2Attention` directly (bypassing the per-class
`get_fuser` cache, so both q_lora variants of the same class are seen)."""
attn = _attention(q_lora_rank)
return MLAFuser.match(trace(attn), attn)
def test_discovers_modules_without_q_lora():
fuser = _match(q_lora_rank=None)
assert isinstance(fuser, MLAFuser)
assert fuser.q_proj_name == "q_proj"
assert fuser.kv_a_proj_name == "kv_a_proj_with_mqa"
assert fuser.kv_a_layernorm_name == "kv_a_layernorm"
assert fuser.kv_b_proj_name == "kv_b_proj"
assert fuser.o_proj_name == "o_proj"
assert fuser.q_a_proj_name is None
# Nothing is stacked without a query LoRA.
assert fuser.packed_modules_mapping == {}
assert fuser.orig_to_new_stacked("model.layers.0.self_attn") == {}
def test_discovers_modules_with_q_lora():
fuser = _match(q_lora_rank=64)
assert isinstance(fuser, MLAFuser)
assert fuser.q_a_proj_name == "q_a_proj"
assert fuser.q_a_layernorm_name == "q_a_layernorm"
assert fuser.q_b_proj_name == "q_b_proj"
assert fuser.kv_a_proj_name == "kv_a_proj_with_mqa"
assert fuser.kv_a_layernorm_name == "kv_a_layernorm"
assert fuser.kv_b_proj_name == "kv_b_proj"
assert fuser.o_proj_name == "o_proj"
assert fuser.q_proj_name is None
def test_q_lora_stacks_qkv_a_proj():
"""The MLA layer reads `q_a_proj` and `kv_a_proj_with_mqa` fused into one
down-projection, so both checkpoint weights must remap into it."""
fuser = _match(q_lora_rank=64)
assert isinstance(fuser, MLAFuser)
prefix = "model.layers.0.self_attn"
merged = f"{prefix}.{_FUSED_QKV_A_PROJ}"
assert fuser.packed_modules_mapping == {
_FUSED_QKV_A_PROJ: ["q_a_proj", "kv_a_proj_with_mqa"]
}
assert fuser.orig_to_new_stacked(prefix) == {
f"{prefix}.q_a_proj": (merged, 0),
f"{prefix}.kv_a_proj_with_mqa": (merged, 1),
}
@pytest.mark.parametrize("q_lora_rank", [64, None])
def test_update_forward_rewrites_real_attention(q_lora_rank):
"""The source rewrite must survive the input-stability check.
The q-LoRA path hoists the merged down-projection to a top-level statement
while its calls sit inside the `q_lora_rank` branches, so the region it
spans is wide; a check that over-approximates aliasing would silently refuse
here and cost the fusion rather than fail loudly."""
fuser = _match(q_lora_rank)
assert isinstance(fuser, MLAFuser)
fuser.update_forward(_attention(q_lora_rank))
names = set(fuser.fused_forward.__code__.co_names)
if fuser.q_a_proj_name is not None:
assert _FUSED_QKV_A_PROJ in names
assert not {"q_a_proj", "kv_a_proj_with_mqa"} & names
class _Norm(nn.Module):
"""A real RMSNorm computation: `match` verifies chain norms via `RMSNormFuser`."""
def __init__(self, n: int):
super().__init__()
self.weight = nn.Parameter(torch.ones(n))
def forward(self, x):
variance = x.pow(2).mean(-1, keepdim=True)
return self.weight * (x * torch.rsqrt(variance + 1e-6))
class RenamedMLA(nn.Module):
"""An MLA-shaped attention whose children have non-standard names, proving
discovery is by structure and not attribute name."""
def __init__(self):
super().__init__()
heads, kv_lora, rope, nope, v, hidden = 4, 32, 8, 8, 16, 64
self.alpha = nn.Linear(hidden, heads * (nope + rope)) # q_proj
self.beta = nn.Linear(hidden, kv_lora + rope) # kv_a_proj_with_mqa
self.gamma = _Norm(kv_lora) # kv_a_layernorm
self.delta = nn.Linear(kv_lora, heads * (nope + v)) # kv_b_proj
self.omega = nn.Linear(heads * v, hidden) # o_proj (uncalled)
self.kv_lora, self.rope = kv_lora, rope
def forward(self, hidden_states):
q = self.alpha(hidden_states)
kv_lora, k_pe = torch.split(
self.beta(hidden_states), [self.kv_lora, self.rope], dim=-1
)
expanded = self.delta(self.gamma(kv_lora))
# Stand-in for the attention interface; `match` finds `o_proj` (omega) from
# the source as the Linear producing the returned value.
attn_output = expanded.sum() + q.sum() + k_pe.sum()
attn_output = self.omega(attn_output)
return attn_output
def test_discovers_modules_under_arbitrary_names():
"""Discovery is purely structural: `RenamedMLA` gives its children non-standard
names, and `match` still locates each projection by dataflow."""
with torch.device("meta"):
module = RenamedMLA()
fuser = MLAFuser.match(trace(module), module)
assert isinstance(fuser, MLAFuser)
assert fuser.q_a_proj_name is None
assert fuser.q_proj_name == "alpha"
assert fuser.kv_a_proj_name == "beta"
assert fuser.kv_a_layernorm_name == "gamma"
assert fuser.kv_b_proj_name == "delta"
assert fuser.o_proj_name == "omega"
class GLU(nn.Module):
"""A gated MLP: matches the GLU fuser, never MLA."""
def __init__(self):
super().__init__()
self.gate = nn.Linear(16, 16)
self.up = nn.Linear(16, 16)
def forward(self, x):
return self.up(F.silu(self.gate(x)))
def test_non_mla_is_not_matched():
with torch.device("meta"):
assert get_fuser(GLU(), MLAFuser) is None
@@ -1,611 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Unit tests for the Transformers modeling backend's MoE fuser."""
import pytest
import torch
import torch.nn as nn
import torch.nn.functional as F
from vllm.model_executor.models.transformers.fusers import MoEBlockFuser
from .test_linear import GLUMLP
class TopKRouter(nn.Module):
"""HF v5 top-k router: `linear -> softmax -> topk (-> renorm)`."""
def __init__(self, num_experts=8, hidden=16, top_k=2, sigmoid=False):
super().__init__()
self.top_k = top_k
self.sigmoid = sigmoid
self.weight = nn.Parameter(torch.zeros(num_experts, hidden))
def forward(self, hidden_states):
logits = F.linear(hidden_states, self.weight)
scores = torch.sigmoid(logits) if self.sigmoid else F.softmax(logits, dim=-1)
value, index = torch.topk(scores, self.top_k, dim=-1)
value = value / value.sum(dim=-1, keepdim=True)
return logits, value, index
class SelectionScoredRouter(TopKRouter):
"""Scores the selected logits only (GraniteMoE/Aria style), in fp32."""
def forward(self, hidden_states):
logits = F.linear(hidden_states, self.weight).float()
value, index = torch.topk(logits, self.top_k, dim=-1)
value = (
torch.sigmoid(value) if self.sigmoid else F.softmax(value, dim=-1)
).type_as(hidden_states)
return logits, value, index
class EpsilonRenormRouter(TopKRouter):
"""Renormalizes via an epsilon-offset denominator (DeepSeek-V3 style)."""
def forward(self, hidden_states):
logits = F.linear(hidden_states, self.weight)
scores = torch.sigmoid(logits)
value, index = torch.topk(scores, self.top_k, dim=-1)
denominator = value.sum(dim=-1, keepdim=True) + 1e-20
value /= denominator
return logits, value, index
class UntraceableRouter(TopKRouter):
"""Scores and selects, then loses the trace before the renormalization."""
def forward(self, hidden_states):
logits = F.linear(hidden_states, self.weight)
scores = F.softmax(logits, dim=-1)
value, index = torch.topk(scores, self.top_k, dim=-1)
if value.sum() > 0: # data-dependent branch: tracing stops here
value = value / value.sum(dim=-1, keepdim=True)
return logits, value, index
class ScaledRouter(TopKRouter):
"""Greedy router scaling its top-k weights (DeepSeek `routed_scaling_factor`)."""
def forward(self, hidden_states):
logits = F.linear(hidden_states, self.weight)
scores = F.softmax(logits, dim=-1)
value, index = torch.topk(scores, self.top_k, dim=-1)
value = value * 16.0
return logits, value, index
class Fp32Router(TopKRouter):
"""Router that computes its logits in fp32 (DeepSeek/GLM style)."""
def forward(self, hidden_states):
logits = F.linear(
hidden_states.type(torch.float32), self.weight.type(torch.float32)
)
scores = F.softmax(logits, dim=-1)
value, index = torch.topk(scores, self.top_k, dim=-1)
return logits, value.to(hidden_states.dtype), index
class GroupedRouter(TopKRouter):
"""Group-limited router (DeepSeek `group_limited_greedy`), scaled weights."""
def forward(self, hidden_states):
logits = F.linear(hidden_states, self.weight)
scores = F.softmax(logits, dim=-1)
group_scores = scores.view(-1, 4, 2).max(dim=-1).values
group_idx = torch.topk(group_scores, k=2, dim=-1)[1]
group_mask = torch.zeros_like(group_scores)
group_mask.scatter_(1, group_idx, 1)
score_mask = group_mask.unsqueeze(-1).expand(-1, 4, 2).reshape(-1, 8)
scores = scores.masked_fill(~score_mask.bool(), 0.0)
value, index = torch.topk(scores, self.top_k, dim=-1)
value = value * 16.0
return logits, value, index
class CorrectionRouter(nn.Module):
"""Router with a score-correction bias buffer (DeepSeek-V3 noaux) -> matched."""
def __init__(self, num_experts=8, hidden=16):
super().__init__()
self.weight = nn.Parameter(torch.zeros(num_experts, hidden))
self.register_buffer("e_score_correction_bias", torch.zeros(num_experts))
def forward(self, hidden_states):
logits = F.linear(hidden_states, self.weight)
scores = torch.sigmoid(logits) + self.e_score_correction_bias
_, index = torch.topk(scores, 2, dim=-1)
return logits, scores, index
class BiasedRouter(TopKRouter):
"""A valid top-k router but not `weight`-only (extra `bias` param) -> declined."""
def __init__(self):
super().__init__()
self.bias = nn.Parameter(torch.zeros(8))
def forward(self, hidden_states):
logits = F.linear(hidden_states, self.weight) + self.bias
scores = F.softmax(logits, dim=-1)
value, index = torch.topk(scores, self.top_k, dim=-1)
return logits, value, index
class DisconnectedRouter(TopKRouter):
"""linear+softmax+top-k present but top-k ignores the logits -> not a router."""
def forward(self, hidden_states):
logits = F.linear(hidden_states, self.weight)
_ = F.softmax(logits, dim=-1) # scored, but not consumed by top-k
value, index = torch.topk(hidden_states, self.top_k, dim=-1)
return logits, value, index
class ExtraInputRouter(TopKRouter):
"""Router taking a block-owned correction bias as a second input (HYV3)."""
def forward(self, hidden_states, bias=0.0):
logits = F.linear(hidden_states, self.weight)
_, index = torch.topk(logits.sigmoid() + bias, self.top_k, dim=-1)
value = logits.sigmoid().gather(1, index)
return logits, value, index
class MoEExperts(nn.Module):
"""Packed experts (3D weights); only its name (`experts`) matters here."""
def __init__(self, num_experts=8, hidden=16, inter=32):
super().__init__()
self.gate_up_proj = nn.Parameter(torch.zeros(num_experts, 2 * inter, hidden))
self.down_proj = nn.Parameter(torch.zeros(num_experts, hidden, inter))
def forward(self, hidden_states, index, weights):
return hidden_states
class MoEBlock(nn.Module):
"""Single-tensor MoE block (Qwen3-style); subclasses override `_shared`."""
def __init__(self, router_cls=TopKRouter):
super().__init__()
self.experts = MoEExperts()
self.gate = router_cls()
def _shared(self, x, logits):
"""The term added to the experts' output (none for a plain block)."""
return None
def forward(self, hidden_states):
x = hidden_states.reshape(-1, hidden_states.shape[-1])
logits, weights, index = self.gate(x)
out = self.experts(x, index, weights)
if (shared := self._shared(x, logits)) is not None:
out = out + shared
return out.reshape(hidden_states.shape)
class MoEBlockNoShared(MoEBlock):
"""No shared-expert child but a gate-derived add -> declined, not misread."""
def _shared(self, x, logits):
return logits.sum()
class MoEBlockShared(MoEBlock):
"""A block with a shared expert and its sigmoid gate (Qwen2-style)."""
def __init__(self):
super().__init__()
self.shared_expert = GLUMLP()
self.shared_expert_gate = nn.Linear(16, 1, bias=False)
def _shared(self, x, logits):
return torch.sigmoid(self.shared_expert_gate(x)) * self.shared_expert(x)
class MoEBlockSharedNoGate(MoEBlock):
"""A block with an ungated shared expert -> native, shared passed through."""
def __init__(self):
super().__init__()
self.shared_expert = GLUMLP()
def _shared(self, x, logits):
return self.shared_expert(x)
class MoEBlockTuple(MoEBlock):
"""A tuple-returning block (gpt-oss-style) -> must decline."""
def forward(self, hidden_states):
x = hidden_states.reshape(-1, hidden_states.shape[-1])
_, weights, index = self.gate(x)
return self.experts(x, index, weights), index
class MoEBlockTupleVar(MoEBlock):
"""Returns a name bound to a tuple, not a literal tuple -> must still decline."""
def forward(self, hidden_states):
x = hidden_states.reshape(-1, hidden_states.shape[-1])
_, weights, index = self.gate(x)
result = self.experts(x, index, weights), index
return result
class MoEBlockNestedTupleReturn(MoEBlock):
"""Tuple `return` in a nested helper; block returns one tensor -> still fuses."""
def forward(self, hidden_states):
def keep(a, b):
return a, b
x = hidden_states.reshape(-1, hidden_states.shape[-1])
_, weights, index = self.gate(x)
out, _ = keep(self.experts(x, index, weights), index)
return out.reshape(hidden_states.shape)
class PlainMLP(nn.Module):
"""A non-GLU FFN: `down(act(up(x)))`, no gating multiply."""
def __init__(self, hidden: int = 16, inter: int = 32):
super().__init__()
self.up_proj = nn.Linear(hidden, inter, bias=False)
self.down_proj = nn.Linear(inter, hidden, bias=False)
self.act_fn = nn.SiLU()
def forward(self, x):
return self.down_proj(self.act_fn(self.up_proj(x)))
class MoEBlockSharedNonGLU(MoEBlock):
"""A non-GLU shared expert -> detected by dataflow (no gate/up merge)."""
def __init__(self):
super().__init__()
self.shared_expert = PlainMLP()
def _shared(self, x, logits):
return self.shared_expert(x)
class MoEBlockUnaccounted(MoEBlock):
"""A weight-bearing child outside the fused dataflow (pre-router) -> declined."""
def __init__(self):
super().__init__()
self.extra = nn.Linear(16, 16, bias=False)
def forward(self, hidden_states):
x = self.extra(hidden_states.reshape(-1, hidden_states.shape[-1]))
_, weights, index = self.gate(x)
return self.experts(x, index, weights).reshape(hidden_states.shape)
class BufferScale(nn.Module):
"""A stateful child carrying only a buffer (no parameters)."""
def __init__(self, hidden: int = 16):
super().__init__()
self.register_buffer("scale", torch.ones(hidden))
def forward(self, x):
return x * self.scale
class MoEBlockUnaccountedBuffer(MoEBlockUnaccounted):
"""Like `MoEBlockUnaccounted`, but the extra child holds only a buffer."""
def __init__(self):
super().__init__()
self.extra = BufferScale()
class MoEBlockExpertsKeywords(MoEBlock):
"""Passes the experts' routing by keyword -> still fuses."""
def forward(self, hidden_states):
x = hidden_states.reshape(-1, hidden_states.shape[-1])
_, weights, index = self.gate(x)
return self.experts(x, weights=weights, index=index)
class ScaledExperts(MoEExperts):
def forward(self, hidden_states, index, weights, scale=1.0):
return hidden_states * scale
class MoEBlockExpertsExtraArg(MoEBlock):
"""Passes the experts an extra argument the fused forward would drop."""
def __init__(self):
super().__init__()
self.experts = ScaledExperts()
def forward(self, hidden_states):
x = hidden_states.reshape(-1, hidden_states.shape[-1])
_, weights, index = self.gate(x)
return self.experts(x, index, weights, scale=2.0)
class MoEBlockSharedUnflattened(MoEBlockSharedNoGate):
"""Gate and shared expert read the unflattened input (Kolibri-1)."""
def forward(self, hidden_states):
_, weights, index = self.gate(hidden_states)
x = hidden_states.view(-1, hidden_states.shape[-1])
out = self.experts(x, index, weights).view(*hidden_states.shape)
return out + self.shared_expert(hidden_states)
class MoEBlockSharedAveraged(MoEBlockSharedUnflattened):
"""Averages the experts' and shared expert's outputs (Cohere2-MoE)."""
def forward(self, hidden_states):
return super().forward(hidden_states) / 2
class MoEBlockScaled(MoEBlockSharedNoGate):
"""Scales the experts' output by `routed_scaling_factor` (Laguna)."""
def forward(self, hidden_states):
x = hidden_states.reshape(-1, hidden_states.shape[-1])
_, weights, index = self.gate(x)
return self.experts(x, index, weights) * 2.5 + self.shared_expert(x)
class MoEBlockScaledBiasedRouter(MoEBlockScaled):
"""An unrecognized router with a scaled experts' output."""
def __init__(self):
super().__init__()
self.gate = BiasedRouter()
class MoEBlockSharedBiasedRouter(MoEBlockSharedUnflattened):
"""An unrecognized router with a shared expert."""
def __init__(self):
super().__init__()
self.gate = BiasedRouter()
class MoEBlockGateExtraInput(MoEBlock):
"""Passes the gate a block-owned bias the fused forward would drop (HYV3)."""
def __init__(self):
super().__init__(ExtraInputRouter)
self.register_buffer("e_score_correction_bias", torch.zeros(8))
def forward(self, hidden_states):
x = hidden_states.reshape(-1, hidden_states.shape[-1])
_, weights, index = self.gate(x, self.e_score_correction_bias)
return self.experts(x, index, weights)
class MoEBlockCastCombine(MoEBlockSharedNoGate):
"""Combines the outputs in fp32 (HYV3)."""
def forward(self, hidden_states):
x = hidden_states.reshape(-1, hidden_states.shape[-1])
_, weights, index = self.gate(x)
out = self.experts(x, index, weights).float() + self.shared_expert(x).float()
return out.to(x.dtype)
class MoEBlockIdentityLatent(MoEBlockSharedNoGate):
"""Wraps the experts in unused (identity) latent projections (NemotronH)."""
def __init__(self):
super().__init__()
self.fc1_latent_proj = nn.Identity()
self.fc2_latent_proj = nn.Identity()
def forward(self, hidden_states):
x = hidden_states.reshape(-1, hidden_states.shape[-1])
_, weights, index = self.gate(x)
out = self.fc2_latent_proj(
self.experts(self.fc1_latent_proj(x), index, weights)
)
return out + self.shared_expert(x)
@pytest.mark.parametrize("sigmoid", [False, True])
def test_moe_fuser_detects_router(sigmoid):
with torch.device("meta"):
block = MoEBlock(lambda: TopKRouter(sigmoid=sigmoid))
fuser = MoEBlockFuser.match(block, "experts")
assert isinstance(fuser, MoEBlockFuser)
assert fuser.gate_name == "gate"
assert fuser.scoring_func == ("sigmoid" if sigmoid else "softmax")
assert fuser.shared_name is None and fuser.shared_gate_name is None
@pytest.mark.parametrize("sigmoid", [False, True])
def test_moe_fuser_matches_router_scoring_the_selection(sigmoid):
"""Scoring the selection rather than all logits still allows the fuser to match."""
with torch.device("meta"):
block = MoEBlock(lambda: SelectionScoredRouter(sigmoid=sigmoid))
fuser = MoEBlockFuser.match(block, "experts")
assert isinstance(fuser, MoEBlockFuser)
assert fuser.gate_name == "gate"
assert fuser.scoring_func == ("sigmoid" if sigmoid else "softmax")
assert fuser.renormalize is (not sigmoid)
assert fuser.router_dtype == torch.float32
def test_moe_fuser_matches_scaled_router():
"""Weight scaling after the top-k (DeepSeek style) does not break matching."""
with torch.device("meta"):
block = MoEBlock(ScaledRouter)
assert isinstance(MoEBlockFuser.match(block, "experts"), MoEBlockFuser)
@pytest.mark.parametrize(
"router_cls,expected",
[
(TopKRouter, True), # divides by the sum of the selection
(EpsilonRenormRouter, True), # ... with an epsilon in the denominator
(ScaledRouter, False), # scales the weights instead of renormalizing
(UntraceableRouter, None), # trace stops before it could say
],
)
def test_moe_fuser_reads_renormalize_from_the_gate(router_cls, expected):
"""The router's graph, not the config, decides whether vLLM renormalizes: it
is the code being replaced, and it already bakes in `norm_topk_prob` because
the flag is a Python bool that fx resolves at trace time. Absence of the
division only means "no" on a complete trace."""
with torch.device("meta"):
block = MoEBlock(router_cls)
fuser = MoEBlockFuser.match(block, "experts")
assert isinstance(fuser, MoEBlockFuser)
assert fuser.renormalize is expected
def test_moe_fuser_matches_grouped_router():
"""Group masking between the score and the routing top-k still matches: the
scorer is anchored on the routing (last) top-k, not the group one."""
with torch.device("meta"):
block = MoEBlock(GroupedRouter)
fuser = MoEBlockFuser.match(block, "experts")
assert isinstance(fuser, MoEBlockFuser)
assert fuser.scoring_func == "softmax"
def test_moe_fuser_matches_correction_router():
"""A score-correction bias buffer (DeepSeek-V3 noaux) is allowed; the
rebuilt gate carries it in fp32 for FusedMoE's biased routers."""
with torch.device("meta"):
block = MoEBlock(CorrectionRouter)
fuser = MoEBlockFuser.match(block, "experts")
assert isinstance(fuser, MoEBlockFuser)
assert fuser.scoring_func == "sigmoid"
def test_moe_fuser_reads_router_dtype_from_the_gate():
"""A router that computes its logits in fp32 must keep routing in fp32 when
rebuilt, even though no config field names the dtype. The cast back to the
activation dtype after the top-k must not be mistaken for the routing dtype."""
with torch.device("meta"):
fp32 = MoEBlockFuser.match(MoEBlock(Fp32Router), "experts")
default = MoEBlockFuser.match(MoEBlock(TopKRouter), "experts")
assert fp32.router_dtype == torch.float32
assert default.router_dtype is None
def test_moe_fuser_detects_shared_experts():
with torch.device("meta"):
block = MoEBlockShared()
fuser = MoEBlockFuser.match(block, "experts")
assert isinstance(fuser, MoEBlockFuser)
assert fuser.shared_name == "shared_expert"
assert fuser.shared_gate_name == "shared_expert_gate"
def test_moe_fuser_shared_without_gate():
with torch.device("meta"):
block = MoEBlockSharedNoGate()
fuser = MoEBlockFuser.match(block, "experts")
assert isinstance(fuser, MoEBlockFuser)
assert fuser.shared_name == "shared_expert"
assert fuser.shared_gate_name is None
def test_moe_fuser_detects_non_glu_shared_expert():
with torch.device("meta"):
block = MoEBlockSharedNonGLU()
fuser = MoEBlockFuser.match(block, "experts")
assert isinstance(fuser, MoEBlockFuser)
# Recognised by dataflow (added to the experts' output), though not a GLU.
assert fuser.shared_name == "shared_expert"
assert fuser.shared_gate_name is None
@pytest.mark.parametrize(
"block_cls",
[
lambda: MoEBlock(BiasedRouter), # router with an unrecognized extra param
MoEBlockTuple, # tuple-returning block (e.g. gpt-oss)
MoEBlockTupleVar, # tuple returned via a name binding, not a literal
MoEBlockUnaccounted, # weight-bearing child outside the fused dataflow
MoEBlockUnaccountedBuffer, # buffer-only child outside the fused dataflow
MoEBlockExpertsExtraArg, # extra experts argument the fused forward drops
MoEBlockNoShared, # gate-derived term added to the experts' output
MoEBlockSharedAveraged, # experts' and shared expert's outputs averaged
MoEBlockGateExtraInput, # gate input the fused forward drops
],
)
def test_moe_fuser_declines_unsupported(block_cls):
with torch.device("meta"):
block = block_cls()
assert MoEBlockFuser.match(block, "experts") is None
def test_moe_fuser_ignores_nested_returns():
"""A tuple `return` inside a nested helper must not decline a block whose own
forward returns a single tensor."""
with torch.device("meta"):
block = MoEBlockNestedTupleReturn()
assert isinstance(MoEBlockFuser.match(block, "experts"), MoEBlockFuser)
def test_moe_fuser_router_requires_connected_dataflow():
"""A gate with linear + softmax + top-k present but not wired as a router
(top-k selects over the input, not the scored logits) is not detected."""
with torch.device("meta"):
block = MoEBlock(DisconnectedRouter)
assert MoEBlockFuser.match(block, "experts") is None
def test_moe_fuser_unrecognized_router_moves_shared_expert():
"""Without vLLM routing, the shared expert still moves to vLLM."""
with torch.device("meta"):
block = MoEBlockSharedBiasedRouter()
fuser = MoEBlockFuser.match(block, "experts")
assert fuser.scoring_func is None
assert fuser.shared_name == "shared_expert"
@pytest.mark.parametrize(
("block_cls", "scoring_func"),
[
(MoEBlockSharedUnflattened, None), # routing falls back to Transformers
(MoEBlock, "softmax"), # no shared output to divide
],
)
def test_moe_fuser_divided_shared_output(block_cls, scoring_func):
with torch.device("meta"):
fuser = MoEBlockFuser.match(block_cls(), "experts", divides_shared_output=True)
assert fuser.scoring_func == scoring_func
def test_moe_fuser_scaled_output_needs_vllm_routing():
"""Only vLLM routing applies the routed scale, so nothing else moves without it."""
with torch.device("meta"):
block = MoEBlockScaledBiasedRouter()
assert MoEBlockFuser.match(block, "experts") is None
def test_moe_fuser_matches_experts_called_by_keyword():
with torch.device("meta"):
block = MoEBlockExpertsKeywords()
assert isinstance(MoEBlockFuser.match(block, "experts"), MoEBlockFuser)
def test_moe_fuser_scaled_output_with_vllm_routing():
"""A scaled block fuses because vLLM routing applies `routed_scaling_factor`."""
with torch.device("meta"):
fuser = MoEBlockFuser.match(MoEBlockScaled(), "experts")
assert fuser.scoring_func == "softmax"
assert fuser.shared_name == "shared_expert"
@pytest.mark.parametrize("block_cls", [MoEBlockCastCombine, MoEBlockIdentityLatent])
def test_moe_fuser_looks_through_casts_and_identity(block_cls):
with torch.device("meta"):
fuser = MoEBlockFuser.match(block_cls(), "experts")
assert fuser.shared_name == "shared_expert"
@@ -1,409 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Unit tests for the Transformers modeling backend's RMSNorm fuser."""
from types import SimpleNamespace
import pytest
import torch
import torch.nn as nn
import torch.nn.functional as F
from vllm.model_executor.models.transformers.fuser import get_fuser
from vllm.model_executor.models.transformers.fusers import RMSNormFuser
class RMSNorm(nn.Module):
"""The canonical HF RMSNorm: `weight * x * rsqrt(mean(x**2) + eps)`."""
def __init__(self, hidden: int = 16, eps: float = 1e-5, weight: bool = True):
super().__init__()
if weight:
self.weight = nn.Parameter(torch.ones(hidden))
self.variance_epsilon = eps
def _rms(self, x):
return x * torch.rsqrt(x.pow(2).mean(-1, keepdim=True) + self.variance_epsilon)
def forward(self, x):
return self.weight * self._rms(x.to(torch.float32)).to(x.dtype)
class GemmaRMSNorm(RMSNorm):
"""Zero-centered weight: `(1 + weight) * normalized`."""
def __init__(self, hidden: int = 16, eps: float = 1e-6):
super().__init__(hidden, eps)
self.weight = nn.Parameter(torch.zeros(hidden))
def forward(self, x):
return (1.0 + self.weight) * self._rms(x.to(torch.float32)).to(x.dtype)
class WeightlessRMSNorm(RMSNorm):
"""No scale parameter (e.g. Gemma3n `with_scale=False`)."""
def __init__(self, hidden: int = 16, eps: float = 1e-6):
super().__init__(hidden, eps, weight=False)
def forward(self, x):
return self._rms(x.to(torch.float32)).to(x.dtype)
class Gemma4RMSNorm(RMSNorm):
"""`pow(v, -0.5)` where the others write `rsqrt(v)` (HF `Gemma4RMSNorm._norm`).
Gemma 4 spells the reciprocal square root as `torch.pow(..., -0.5)` to keep
Torch and JAX in agreement, and -- unlike Gemma 1-3 -- its weight is *not*
zero-centered, so this fuses to a plain `RMSNorm`.
"""
def _rms(self, x):
mean_squared = x.pow(2).mean(-1, keepdim=True) + self.variance_epsilon
return x * torch.pow(mean_squared, -0.5)
def forward(self, x):
return (self._rms(x.float()) * self.weight.float()).type_as(x)
class PowOperatorRMSNorm(Gemma4RMSNorm):
"""The `v ** -0.5` operator spelling of the same reciprocal square root."""
def _rms(self, x):
return x * (x.pow(2).mean(-1, keepdim=True) + self.variance_epsilon) ** -0.5
class KwargPowRMSNorm(Gemma4RMSNorm):
"""The keyword spelling `torch.pow(v, exponent=-0.5)` of the same rsqrt."""
def _rms(self, x):
mean_squared = x.pow(2).mean(-1, keepdim=True) + self.variance_epsilon
return x * torch.pow(mean_squared, exponent=-0.5)
class KwargBasePowRMSNorm(Gemma4RMSNorm):
"""The fully functional spelling `torch.pow(input=v, exponent=-0.5)`."""
def _rms(self, x):
mean_squared = x.pow(2).mean(-1, keepdim=True) + self.variance_epsilon
return x * torch.pow(input=mean_squared, exponent=-0.5)
class WeightlessGemma4RMSNorm(Gemma4RMSNorm):
"""Gemma 4 with `with_scale=False`: the `pow` spelling and no scale parameter."""
def __init__(self, hidden: int = 16, eps: float = 1e-6):
super().__init__(hidden, eps, weight=False)
def forward(self, x):
return self._rms(x.float()).type_as(x)
class LayerNorm(RMSNorm):
"""An RMSNorm not named `*RMSNorm`, keeping the input dtype (no upcast)."""
def __init__(self, hidden: int = 16, eps: float = 1e-6):
super().__init__(hidden, eps)
def forward(self, x):
return self.weight * self._rms(x)
class NamedEpsRMSNorm(nn.Module):
"""Holds eps under `attr`, with no `variance_epsilon` to find instead."""
attr = "eps"
def __init__(self, hidden: int = 16, eps: float = 1e-6):
super().__init__()
self.weight = nn.Parameter(torch.ones(hidden))
setattr(self, self.attr, eps)
def forward(self, x):
eps = getattr(self, self.attr)
return self.weight * (x * torch.rsqrt(x.pow(2).mean(-1, True) + eps))
class ToleranceRMSNorm(NamedEpsRMSNorm):
"""Names eps something other than `eps`, so only its value identifies it."""
attr = "tolerance"
class LiteralEpsRMSNorm(RMSNorm):
"""eps is a literal in the source, and an unrelated attribute happens to
hold the same value: matching on the value alone would bind to it."""
def __init__(self, hidden: int = 16, eps: float = 1e-5):
super().__init__(hidden, eps)
self.epsilon_lookalike = 1e-4
def forward(self, x):
return self.weight * (x * torch.rsqrt(x.pow(2).mean(-1, True) + 1e-4))
class AmbiguousEpsRMSNorm(nn.Module):
"""Two attributes hold the eps value, and the forward reads the second, so
their order cannot pick it."""
def __init__(self, hidden: int = 16, eps: float = 1e-6):
super().__init__()
self.weight = nn.Parameter(torch.ones(hidden))
self.decoy_eps = eps
self.variance_epsilon = eps
def forward(self, x):
var = x.pow(2).mean(-1, True)
return self.weight * (x * torch.rsqrt(var + self.variance_epsilon))
class NotAnRMSNorm(RMSNorm):
"""Mean-subtracting LayerNorm-like math -> not an RMSNorm."""
def __init__(self, hidden: int = 16, eps: float = 1e-6):
super().__init__(hidden, eps)
def forward(self, x):
x = x - x.mean(-1, keepdim=True)
variance = x.var(-1, keepdim=True)
return self.weight * x / torch.sqrt(variance + self.variance_epsilon)
class GatedRMSNorm(RMSNorm):
"""Second input and tail compute -> not an RMSNorm."""
def forward(self, x, gate=None):
normed = self.weight * self._rms(x.to(torch.float32)).to(x.dtype)
return normed * F.silu(gate)
class GatedFusedRMSNorm(nn.Module):
"""Same as GatedRMSNorm, but built on the fused `rms_norm` op -> not an RMSNorm."""
def __init__(self, hidden: int = 16, eps: float = 1e-5):
super().__init__()
self.weight = nn.Parameter(torch.ones(hidden))
self.eps = eps
def forward(self, x, gate=None):
return F.rms_norm(x, (x.shape[-1],), self.weight, self.eps) * F.silu(gate)
class UntraceableGatedRMSNorm(RMSNorm):
"""Tracer can't see tail compute in forward, but still has a second input (gate)."""
def forward(self, x, gate=None):
normed = self.weight * self._rms(x.to(torch.float32)).to(x.dtype)
if gate.sum() > 0: # untraceable -> partial graph, no visible tail
normed = normed * F.silu(gate)
return normed
@pytest.mark.parametrize(
"cls,eps,zero_centered",
[
(RMSNorm, 1e-5, False),
(GemmaRMSNorm, 1e-6, True),
(WeightlessRMSNorm, 1e-6, False),
(Gemma4RMSNorm, 1e-6, False),
(PowOperatorRMSNorm, 1e-6, False),
(KwargPowRMSNorm, 1e-6, False),
(KwargBasePowRMSNorm, 1e-6, False),
(WeightlessGemma4RMSNorm, 1e-6, False),
(LayerNorm, 1e-6, False),
(torch.nn.RMSNorm, 1e-5, False), # fused `F.rms_norm` op
],
)
def test_detects_rms_norm_variants(cls, eps, zero_centered):
with torch.device("meta"):
fuser = get_fuser(cls(16, eps=eps), RMSNormFuser)
assert isinstance(fuser, RMSNormFuser)
assert fuser.zero_centered == zero_centered
@pytest.mark.parametrize("cls", [NotAnRMSNorm, nn.LayerNorm, nn.SiLU])
def test_non_rms_norms_are_not_matched(cls):
with torch.device("meta"):
module = cls(16) if cls is nn.LayerNorm else cls()
assert get_fuser(module, RMSNormFuser) is None
@pytest.mark.parametrize(
"cls", [GatedRMSNorm, GatedFusedRMSNorm, UntraceableGatedRMSNorm]
)
def test_gated_rms_norm_is_not_fused(cls):
with torch.device("meta"):
assert get_fuser(cls(), RMSNormFuser) is None
@pytest.mark.parametrize(
"cls,expected,zero_centered",
[
(RMSNorm, "RMSNorm", False),
(GemmaRMSNorm, "GemmaRMSNorm", True),
(WeightlessRMSNorm, "RMSNorm", False),
(Gemma4RMSNorm, "RMSNorm", False),
(KwargPowRMSNorm, "RMSNorm", False),
(KwargBasePowRMSNorm, "RMSNorm", False),
(WeightlessGemma4RMSNorm, "RMSNorm", False),
],
)
def test_rms_norm_builds_vllm_class(cls, expected, zero_centered, default_vllm_config):
from vllm.model_executor.layers.layernorm import GemmaRMSNorm as VLLMGemmaRMSNorm
from vllm.model_executor.layers.layernorm import RMSNorm as VLLMRMSNorm
with torch.device("meta"):
module = cls()
fuser = get_fuser(module, RMSNormFuser)
built = fuser.fuse(module, "norm", default_vllm_config)
from vllm.model_executor.models.transformers.fusers.rms_norm import (
TPAwareNormMixin,
)
types_by_name = {"RMSNorm": VLLMRMSNorm, "GemmaRMSNorm": VLLMGemmaRMSNorm}
assert isinstance(built, types_by_name[expected])
assert isinstance(built, TPAwareNormMixin)
assert built.variance_epsilon == module.variance_epsilon
# The weight states the hidden size (and is what the TP check reads); a norm
# without one has none to state.
weight = getattr(module, "weight", None)
assert isinstance(built.weight, nn.Parameter) == (weight is not None)
assert built.weight.shape[0] == (weight.size(0) if weight is not None else 0)
@pytest.mark.parametrize(
"cls", [Gemma4RMSNorm, PowOperatorRMSNorm, KwargPowRMSNorm, KwargBasePowRMSNorm]
)
def test_pow_spelled_norm_fuses_to_equivalent_math(cls, default_vllm_config):
"""Matching `pow(v, -0.5)` is only sound if it really is the reciprocal square
root: the fused norm must reproduce the unfused forward, not just replace it.
`pow` and `rsqrt` take different paths to the same value, so compare at float32
tolerance rather than bit-for-bit."""
torch.manual_seed(0)
module = cls(16)
with torch.no_grad():
module.weight.copy_(torch.randn(16))
built = get_fuser(module, RMSNormFuser).fuse(module, "norm", default_vllm_config)
with torch.no_grad():
built.weight.copy_(module.weight)
x = torch.randn(4, 16)
torch.testing.assert_close(built(x), module(x), rtol=1e-5, atol=1e-6)
def test_weightless_norm_has_no_hidden_size(default_vllm_config):
"""A weightless norm states no hidden size, and the model's (LM) hidden size
would be wrong for one on a sub-dimension: Llama 4's `qk_norm` normalizes
head_dim, so sizing it at hidden_size made the TP gather reject its input.
It fuses with none, and so normalizes the width it is given at any TP size,
matching the unfused norm and vLLM's native `RMSNorm(hidden_size=head_dim)`.
"""
module = WeightlessRMSNorm(72)
built = get_fuser(module, RMSNormFuser).fuse(module, "norm", default_vllm_config)
assert built.hidden_size == 0
built.tp_size = 2 # emulate TP=2 without a real process group
x = torch.randn(4, 72)
torch.testing.assert_close(built(x), module(x))
def test_fused_rms_norm_op_default_eps(default_vllm_config):
"""`torch.nn.RMSNorm` (a single `F.rms_norm` call) matches via the fast path;
its default `eps=None` resolves to `finfo(dtype).eps` in `fuse`."""
from vllm.model_executor.layers.layernorm import RMSNorm as VLLMRMSNorm
with torch.device("meta"):
module = torch.nn.RMSNorm(16) # forward is a single `F.rms_norm` call
fuser = get_fuser(module, RMSNormFuser)
assert isinstance(fuser, RMSNormFuser)
assert not fuser.zero_centered
vllm_config = SimpleNamespace(model_config=SimpleNamespace(dtype=torch.float32))
built = fuser.fuse(module, "norm", vllm_config)
assert isinstance(built, VLLMRMSNorm)
assert built.variance_epsilon == torch.finfo(torch.float32).eps
def test_eps_is_derived_per_instance(default_vllm_config):
"""Two instances of the same norm class with different eps must fuse to their
own eps: the type-cached fuser holds only structure, not this value."""
with torch.device("meta"):
for eps in (1e-5, 1e-6):
module = RMSNorm(16, eps=eps)
fuser = get_fuser(module, RMSNormFuser)
built = fuser.fuse(module, "norm", default_vllm_config)
assert built.variance_epsilon == eps
@pytest.mark.parametrize("cls", [NamedEpsRMSNorm, ToleranceRMSNorm])
def test_eps_attr_is_found_by_value_not_name(cls, default_vllm_config):
"""The eps attribute is identified by holding the traced value, so a norm
stays per-instance correct whatever it names it."""
with torch.device("meta"):
for eps in (1e-5, 1e-6):
module = cls(16, eps=eps)
fuser = get_fuser(module, RMSNormFuser)
assert fuser.eps_attr == cls.attr
built = fuser.fuse(module, "norm", default_vllm_config)
assert built.variance_epsilon == eps
def test_literal_eps_is_not_mistaken_for_an_attribute(default_vllm_config, caplog):
"""A literal eps is recognised as coming from no attribute, even when one
holds the same value, and is taken from the traced source instead."""
logger = "vllm.model_executor.models.transformers.fusers.rms_norm"
with caplog.at_level("DEBUG", logger=logger), torch.device("meta"):
module = LiteralEpsRMSNorm()
fuser = get_fuser(module, RMSNormFuser)
built = fuser.fuse(module, "norm", default_vllm_config)
assert fuser.eps_attr is None
assert built.variance_epsilon == 1e-4
assert "does not hold its eps" in caplog.text
def test_ambiguous_eps_attrs_are_disambiguated(default_vllm_config):
"""When several attributes hold the eps value, the one the forward actually
reads is identified, and they are all left as they were found."""
with torch.device("meta"):
module = AmbiguousEpsRMSNorm(16, eps=1e-6)
before = dict(vars(module))
fuser = get_fuser(module, RMSNormFuser)
assert fuser.eps_attr == "variance_epsilon"
assert vars(module) == before
other = AmbiguousEpsRMSNorm(16, eps=1e-5)
other.decoy_eps = 1e-3
built = fuser.fuse(other, "norm", default_vllm_config)
assert built.variance_epsilon == 1e-5
def test_fused_norm_is_gather_capable(default_vllm_config):
"""Every weighted fused norm is emitted gather-capable, so a norm on a head-sharded
projection (OLMoE-style) self-corrects at runtime with no QKV-specific
plumbing. A full-width input skips the gather and equals a plain norm."""
from vllm.model_executor.layers.layernorm import GemmaRMSNorm, RMSNorm
from vllm.model_executor.models.transformers.fusers import rms_norm
torch.manual_seed(0)
x = torch.randn(4, 16)
for gathered_cls, plain_cls in [
(rms_norm.TPAwareRMSNorm, RMSNorm),
(rms_norm.TPAwareGemmaRMSNorm, GemmaRMSNorm),
]:
gathered = gathered_cls(hidden_size=16, eps=1e-6)
assert isinstance(gathered, rms_norm.TPAwareNormMixin)
plain = plain_cls(hidden_size=16, eps=1e-6)
with torch.no_grad():
weight = torch.randn(16)
gathered.weight.copy_(weight)
plain.weight.copy_(weight)
torch.testing.assert_close(gathered(x), plain(x))
def test_gathered_norm_rejects_uneven_sharding(default_vllm_config):
"""A sharded input (narrower than the full-width weight) that does not tile
the weight evenly across ranks is rejected before any collective."""
from vllm.model_executor.models.transformers.fusers import rms_norm
norm = rms_norm.TPAwareRMSNorm(hidden_size=8, eps=1e-6)
norm.tp_size = 2 # emulate TP=2 without a real process group
with pytest.raises(ValueError, match="does not tile it evenly"):
norm(torch.randn(2, 3)) # 3 * 2 != 8