Fix MLA padding and grouped topk routing in the Transformers modelling backend (#49982)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Co-authored-by: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Harry Mellor
2026-07-27 18:32:32 +00:00
committed by GitHub
co-authored by Claude Opus 5
parent bf2b45b5d6
commit 99de48e98f
5 changed files with 227 additions and 39 deletions
+80 -3
View File
@@ -29,8 +29,48 @@ class TopKRouter(nn.Module):
return logits, value, index
class ScaledRouter(TopKRouter):
"""Greedy router scaling its top-k weights (DeepSeek `routed_scaling_factor`)."""
def forward(self, hidden_states):
logits = F.linear(hidden_states, self.weight)
scores = F.softmax(logits, dim=-1)
value, index = torch.topk(scores, self.top_k, dim=-1)
value = value * 16.0
return logits, value, index
class Fp32Router(TopKRouter):
"""Router that computes its logits in fp32 (DeepSeek/GLM style)."""
def forward(self, hidden_states):
logits = F.linear(
hidden_states.type(torch.float32), self.weight.type(torch.float32)
)
scores = F.softmax(logits, dim=-1)
value, index = torch.topk(scores, self.top_k, dim=-1)
return logits, value.to(hidden_states.dtype), index
class GroupedRouter(TopKRouter):
"""Group-limited router (DeepSeek `group_limited_greedy`), scaled weights."""
def forward(self, hidden_states):
logits = F.linear(hidden_states, self.weight)
scores = F.softmax(logits, dim=-1)
group_scores = scores.view(-1, 4, 2).max(dim=-1).values
group_idx = torch.topk(group_scores, k=2, dim=-1)[1]
group_mask = torch.zeros_like(group_scores)
group_mask.scatter_(1, group_idx, 1)
score_mask = group_mask.unsqueeze(-1).expand(-1, 4, 2).reshape(-1, 8)
scores = scores.masked_fill(~score_mask.bool(), 0.0)
value, index = torch.topk(scores, self.top_k, dim=-1)
value = value * 16.0
return logits, value, index
class CorrectionRouter(nn.Module):
"""Grouped router with a score-correction bias buffer (DeepSeek-V3) -> declined."""
"""Router with a score-correction bias buffer (DeepSeek-V3 noaux) -> matched."""
def __init__(self, num_experts=8, hidden=16):
super().__init__()
@@ -228,6 +268,44 @@ def test_moe_fuser_detects_router(sigmoid):
assert fuser.shared_name is None and fuser.shared_gate_name is None
def test_moe_fuser_matches_scaled_router():
"""Weight scaling after the top-k (DeepSeek style) does not break matching."""
with torch.device("meta"):
block = MoEBlock(ScaledRouter)
assert isinstance(MoEBlockFuser.match(block, "experts"), MoEBlockFuser)
def test_moe_fuser_matches_grouped_router():
"""Group masking between the score and the routing top-k still matches: the
scorer is anchored on the routing (last) top-k, not the group one."""
with torch.device("meta"):
block = MoEBlock(GroupedRouter)
fuser = MoEBlockFuser.match(block, "experts")
assert isinstance(fuser, MoEBlockFuser)
assert fuser.scoring_func == "softmax"
def test_moe_fuser_matches_correction_router():
"""A score-correction bias buffer (DeepSeek-V3 noaux) is allowed; the
rebuilt gate carries it in fp32 for FusedMoE's biased routers."""
with torch.device("meta"):
block = MoEBlock(CorrectionRouter)
fuser = MoEBlockFuser.match(block, "experts")
assert isinstance(fuser, MoEBlockFuser)
assert fuser.scoring_func == "sigmoid"
def test_moe_fuser_reads_router_dtype_from_the_gate():
"""A router that computes its logits in fp32 must keep routing in fp32 when
rebuilt, even though no config field names the dtype. The cast back to the
activation dtype after the top-k must not be mistaken for the routing dtype."""
with torch.device("meta"):
fp32 = MoEBlockFuser.match(MoEBlock(Fp32Router), "experts")
default = MoEBlockFuser.match(MoEBlock(TopKRouter), "experts")
assert fp32.router_dtype == torch.float32
assert default.router_dtype is None
def test_moe_fuser_detects_shared_experts():
with torch.device("meta"):
block = MoEBlockShared()
@@ -269,8 +347,7 @@ def test_moe_fuser_detects_non_glu_shared_expert():
@pytest.mark.parametrize(
"block_cls",
[
lambda: MoEBlock(CorrectionRouter), # score-correction buffer (grouped)
lambda: MoEBlock(BiasedRouter), # router not weight-only (extra param)
lambda: MoEBlock(BiasedRouter), # router with an unrecognized extra param
MoEBlockTuple, # tuple-returning block (e.g. gpt-oss)
MoEBlockTupleVar, # tuple returned via a name binding, not a literal
MoEBlockUnaccounted, # weight-bearing child outside the fused dataflow
@@ -18,6 +18,7 @@
from typing import TYPE_CHECKING
import torch.nn.functional as F
from transformers.modeling_utils import ALL_ATTENTION_FUNCTIONS
from vllm.model_executor.models.transformers.base import Base
@@ -59,9 +60,19 @@ def vllm_attention_forward(
if scaling is not None:
self_attn.impl.scale = float(scaling)
hidden = query.shape[-2]
head_dim_qk = query.shape[-1]
head_dim_v = value.shape[-1]
query, key, value = (x.transpose(1, 2) for x in (query, key, value))
query, key, value = (x.reshape(hidden, -1) for x in (query, key, value))
return self_attn.forward(query, key, value), None
# Pad `value` up to the query/key head size when they differ (expanded MLA).
if head_dim_v != head_dim_qk:
value = F.pad(value.view(-1, head_dim_v), (0, head_dim_qk - head_dim_v))
value = value.reshape(hidden, -1)
attn_output = self_attn.forward(query, key, value)
if head_dim_v != head_dim_qk:
attn_output = attn_output.view(-1, head_dim_qk)[..., :head_dim_v]
attn_output = attn_output.reshape(hidden, -1)
return attn_output, None
ALL_ATTENTION_FUNCTIONS["vllm"] = vllm_attention_forward
@@ -6,14 +6,14 @@ import ast
import inspect
import textwrap
import types
from collections.abc import Iterator
from collections.abc import Iterable, Iterator
from dataclasses import dataclass
from itertools import chain
import torch
from torch import fx, nn
from vllm.distributed import tensor_model_parallel_all_gather
from vllm.model_executor.layers.fused_moe import GateLinear
from vllm.model_executor.layers.linear import ReplicatedLinear
from vllm.model_executor.models.transformers.fx_utils import (
find_node,
@@ -21,14 +21,10 @@ from vllm.model_executor.models.transformers.fx_utils import (
peel,
trace,
)
from vllm.model_executor.models.transformers.utils import named_state
from vllm.model_executor.models.utils import maybe_prefix, sequence_parallel_chunk
def named_state(module: nn.Module) -> Iterator[tuple[str, torch.Tensor]]:
"""`module`'s own state (i.e. named parameters and buffers)."""
return chain(module.named_parameters(), module.named_buffers())
def _own_returns(node: ast.AST) -> Iterator[ast.Return]:
"""`return` statements in `node`'s own scope, not in nested functions."""
stack = list(ast.iter_child_nodes(node))
@@ -79,6 +75,25 @@ def _is_scalar_gate(module: nn.Module) -> bool:
)
def _forced_dtype(nodes: Iterable[fx.Node]) -> torch.dtype | None:
"""The floating dtype `nodes` cast to, if any.
Computations that must run in higher precision say so in their forward (e.g.
`hidden_states.type(torch.float32)`), so the dtype is readable from the graph
even when the config does not name it."""
for node in nodes:
if node.op not in ("call_method", "call_function"):
continue
name = str(node.target).rsplit(".", 1)[-1]
if name == "float":
return torch.float32
if name in ("to", "type"):
for arg in (*node.args[1:], *node.kwargs.values()):
if isinstance(arg, torch.dtype) and arg.is_floating_point:
return arg
return None
def _reaches(node: fx.Node, key: str) -> set[fx.Node]:
"""Returns the set of nodes reachable from `node` by following `key` edges."""
seen: set[fx.Node] = set()
@@ -131,18 +146,24 @@ class MoEBlockFuser:
scoring_func: str
shared_name: str | None
shared_gate_name: str | None
router_dtype: torch.dtype | None = None
@staticmethod
def _match_router(gate: nn.Module) -> str | None:
"""Matches `topk(score(linear(x)))`, `score` being `softmax`/`sigmoid`."""
if [name for name, _ in named_state(gate)] != ["weight"]:
def _match_router(gate: nn.Module) -> tuple[str, torch.dtype | None] | None:
"""Matches `topk(score(linear(x)))`, `score` being `softmax`/`sigmoid`.
Returns the scoring function and the dtype the router computes in."""
state = {name for name, _ in named_state(gate)}
if "weight" not in state or state - {"weight", "e_score_correction_bias"}:
return None
graph = trace(gate)
if graph is None:
return None
topk = find_node(graph, lambda n: is_op(n, "topk"))
if topk is None:
# The routing top-k is the last one; any earlier one scores expert groups.
topks = [node for node in graph.nodes if is_op(node, "topk")]
if not topks:
return None
topk = topks[-1]
# Exactly one scoring op upstream of the top-k, fed (transitively) by a linear.
scorers = [
n
@@ -152,9 +173,11 @@ class MoEBlockFuser:
if len(scorers) != 1:
return None
scorer = scorers[0]
if not any(is_op(n, "linear") for n in _reaches(scorer, "all_input_nodes")):
logits_cone = _reaches(scorer, "all_input_nodes")
if not any(is_op(n, "linear") for n in logits_cone):
return None
return "softmax" if is_op(scorer, "softmax") else "sigmoid"
scoring_func = "softmax" if is_op(scorer, "softmax") else "sigmoid"
return scoring_func, _forced_dtype(logits_cone)
@staticmethod
def _match_shared_experts(
@@ -198,10 +221,14 @@ class MoEBlockFuser:
if _returns_tuple(type(moe_block)):
return None
# Router: the child that scores + top-k selects.
gate_name = scoring_func = None
gate_name = scoring_func = router_dtype = None
for name, child in moe_block.named_children():
if name != experts_name and (func := cls._match_router(child)) is not None:
gate_name, scoring_func = name, func
if (
name != experts_name
and (router := cls._match_router(child)) is not None
):
gate_name = name
scoring_func, router_dtype = router
break
if gate_name is None or scoring_func is None:
return None
@@ -229,17 +256,23 @@ class MoEBlockFuser:
for name, child in moe_block.named_children():
if name not in accounted and next(named_state(child), None) is not None:
return None
return cls(gate_name, scoring_func, shared_name, shared_gate_name)
return cls(gate_name, scoring_func, shared_name, shared_gate_name, router_dtype)
def gate(self, moe_block: nn.Module, prefix: str) -> ReplicatedLinear:
"""Rebuild the HF gate as a `ReplicatedLinear` for vLLM's fused MoE."""
num_experts, hidden_size = getattr(moe_block, self.gate_name).weight.shape
gate = ReplicatedLinear(
def gate(
self, moe_block: nn.Module, prefix: str, out_dtype: torch.dtype | None = None
) -> GateLinear:
"""Rebuild the HF gate as a `GateLinear` for vLLM's fused MoE."""
hf_gate = getattr(moe_block, self.gate_name)
num_experts, hidden_size = hf_gate.weight.shape
gate = GateLinear(
hidden_size,
num_experts,
bias=False,
out_dtype=out_dtype or self.router_dtype,
prefix=maybe_prefix(prefix, self.gate_name),
)
if (bias := getattr(hf_gate, "e_score_correction_bias", None)) is not None:
gate.register_buffer("e_score_correction_bias", bias)
setattr(moe_block, self.gate_name, gate)
return gate
+73 -13
View File
@@ -23,6 +23,7 @@ from typing import TYPE_CHECKING, Any
import torch
import torch.nn as nn
from vllm._aiter_ops import rocm_aiter_ops
from vllm.config.utils import getattr_iter
from vllm.distributed import get_dp_group, get_ep_group
from vllm.forward_context import ForwardContext, get_forward_context
@@ -30,9 +31,10 @@ from vllm.logger import init_logger
from vllm.model_executor.custom_op import PluggableLayer
from vllm.model_executor.layers.fused_moe import FusedMoE, MoERunner, RoutedExperts
from vllm.model_executor.models.interfaces import MixtureOfExperts
from vllm.model_executor.models.transformers.fuser import get_fuser
from vllm.model_executor.models.transformers.fusers.moe import MoEBlockFuser
from vllm.model_executor.models.utils import maybe_prefix
from vllm.utils.torch_utils import direct_register_custom_op
from vllm.utils.torch_utils import STR_DTYPE_TO_TORCH_DTYPE, direct_register_custom_op
from .utils import log_replacement
@@ -176,17 +178,31 @@ class MoEMixin(MixtureOfExperts):
0,
)
# Unused kwargs since we use custom_routing_function:
# - `scoring_func` and `e_score_correction_bias` only used for grouped
# topk routing inside vLLM and are non-trivial to infer
# and hard code `use_grouped_topk=False`
# - `renormalize` passed anyway because it's easy to infer
# - `num_expert_group` and `topk_group` used for inferring expert
# placement strategy in FusedMoE
# - `apply_router_weight_on_input` is already applied in Transformers
# Common kwargs
renormalize = getattr(text_config, "norm_topk_prob", top_k > 1)
# Routed scaling factor kwargs
routed_scaling_factor = getattr(text_config, "routed_scaling_factor", 1.0)
# aiter applies routed_scaling_factor internally
apply_routed_scale_to_output = not rocm_aiter_ops.is_fused_moe_enabled()
routed_scaling_factor_kwargs = dict(
routed_scaling_factor=routed_scaling_factor,
apply_routed_scale_to_output=apply_routed_scale_to_output,
)
# Dtype the router computes in, if it is not the activation dtype.
config_router_dtype = getattr(text_config, "moe_router_dtype", None)
config_router_dtype = STR_DTYPE_TO_TORCH_DTYPE.get(config_router_dtype)
# Grouped topk routing kwargs
num_expert_group = getattr(text_config, "n_group", None)
topk_group = getattr(text_config, "topk_group", None)
use_grouped_topk = num_expert_group is not None and topk_group is not None
grouped_topk_routing_kwargs = dict(
use_grouped_topk=use_grouped_topk,
num_expert_group=num_expert_group,
topk_group=topk_group,
)
# MoE activation function
activation = "silu"
@@ -211,6 +227,9 @@ class MoEMixin(MixtureOfExperts):
self.num_shared_experts = num_shared_experts
self.num_redundant_experts = num_redundant_experts
# Down projections of shared experts consumed by FusedMoE
shared_down_projs: list[tuple[nn.Module, str]] = []
# Recursively fuse MoE layers
def _recursive_replace(module: nn.Module, prefix: str):
for child_name, child_module in module.named_children():
@@ -249,7 +268,6 @@ class MoEMixin(MixtureOfExperts):
hidden_size=hidden_size,
intermediate_size=intermediate_size,
renormalize=renormalize,
use_grouped_topk=False,
quant_config=self.quant_config,
prefix=qual_name,
activation=activation,
@@ -259,17 +277,55 @@ class MoEMixin(MixtureOfExperts):
routed_experts_cls=TransformersRoutedExperts,
)
fuser = MoEBlockFuser.match(moe_block, experts_name)
if self.num_expert_groups <= 1 and fuser is not None:
# _maybe_apply_routed_scale_to_output edge case. Transformers
# decoder layers do not compensate for dividing by scaling factor.
reaches_fp16_trick = (
routed_scaling_factor != 1.0
and apply_routed_scale_to_output
and self.model_config.dtype == torch.float16
and fuser is not None
and fuser.shared_name is not None
)
if reaches_fp16_trick:
logger.warning_once(
"%s could be fused but routing it in vLLM would apply "
"`routed_scaling_factor` by dividing the shared expert "
"output, which only fp16 overflow protection expects the "
"decoder layer to compensate for. Falling back to routing "
"in Transformers; run in bfloat16 to fuse it.",
moe_block_cls,
)
if fuser is not None and not reaches_fp16_trick:
# MoE block forward is fully replaced.
# gate/router and shared expert (if any) runs in FusedMoE.
shared_experts = fuser.shared_experts(moe_block, prefix)
# Store shared experts for later down projection adjustment
if shared_experts is not None:
hf_shared = shared_experts.shared_experts
glu_fuser = get_fuser(hf_shared)
down_name = getattr(glu_fuser, "down_name", None)
if down_name is not None:
shared_down_projs.append((hf_shared, down_name))
# Prefer config, otherwise read it from fuser.
router_dtype = config_router_dtype or fuser.router_dtype
gate = fuser.gate(moe_block, prefix, router_dtype)
kwargs |= dict(
scoring_func=fuser.scoring_func,
is_sequence_parallel=(
self.parallel_config.use_sequence_parallel_moe
),
gate=fuser.gate(moe_block, prefix),
shared_experts=fuser.shared_experts(moe_block, prefix),
gate=gate,
shared_experts=shared_experts,
)
if router_dtype is not None:
kwargs["router_logits_dtype"] = router_dtype
if use_grouped_topk:
kwargs |= grouped_topk_routing_kwargs
if routed_scaling_factor != 1.0:
kwargs |= routed_scaling_factor_kwargs
bias = getattr(gate, "e_score_correction_bias", None)
if bias is not None:
kwargs["e_score_correction_bias"] = bias
fuser.rewrite_forward(moe_block)
routed = "gate + experts"
if fuser.shared_name:
@@ -333,3 +389,7 @@ class MoEMixin(MixtureOfExperts):
self.num_moe_layers = len(self.moe_layers)
# Continue with the replacement of layers in Base
super().recursive_replace()
# GLUFuser likely fused shared_experts. The down projection after the GLU
# normally immediately reduces but we want FusedMoE to handle the reduction.
for hf_shared, down_name in shared_down_projs:
hf_shared.get_submodule(down_name).reduce_results = False
@@ -16,7 +16,9 @@
# limitations under the License.
"""Transformers modeling backend utilities."""
from collections.abc import Iterator
from contextlib import contextmanager
from itertools import chain
from pathlib import Path
from typing import TYPE_CHECKING, Literal
@@ -209,6 +211,11 @@ def recursive_replace_linear(
_recursive_replace(model, prefix=prefix)
def named_state(module: nn.Module) -> Iterator[tuple[str, torch.Tensor]]:
"""`module`'s own state (i.e. named parameters and buffers)."""
return chain(module.named_parameters(), module.named_buffers())
def log_replacement(name: str, old_module: nn.Module, new_module: nn.Module):
logger.debug("%s: %s -> %s", name, old_module, new_module)