Fix MLA padding and grouped topk routing in the Transformers modelling backend (#49982)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Co-authored-by: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Harry Mellor
2026-07-27 18:32:32 +00:00
committed by GitHub
co-authored by Claude Opus 5
parent bf2b45b5d6
commit 99de48e98f
5 changed files with 227 additions and 39 deletions
@@ -18,6 +18,7 @@
from typing import TYPE_CHECKING
import torch.nn.functional as F
from transformers.modeling_utils import ALL_ATTENTION_FUNCTIONS
from vllm.model_executor.models.transformers.base import Base
@@ -59,9 +60,19 @@ def vllm_attention_forward(
if scaling is not None:
self_attn.impl.scale = float(scaling)
hidden = query.shape[-2]
head_dim_qk = query.shape[-1]
head_dim_v = value.shape[-1]
query, key, value = (x.transpose(1, 2) for x in (query, key, value))
query, key, value = (x.reshape(hidden, -1) for x in (query, key, value))
return self_attn.forward(query, key, value), None
# Pad `value` up to the query/key head size when they differ (expanded MLA).
if head_dim_v != head_dim_qk:
value = F.pad(value.view(-1, head_dim_v), (0, head_dim_qk - head_dim_v))
value = value.reshape(hidden, -1)
attn_output = self_attn.forward(query, key, value)
if head_dim_v != head_dim_qk:
attn_output = attn_output.view(-1, head_dim_qk)[..., :head_dim_v]
attn_output = attn_output.reshape(hidden, -1)
return attn_output, None
ALL_ATTENTION_FUNCTIONS["vllm"] = vllm_attention_forward
@@ -6,14 +6,14 @@ import ast
import inspect
import textwrap
import types
from collections.abc import Iterator
from collections.abc import Iterable, Iterator
from dataclasses import dataclass
from itertools import chain
import torch
from torch import fx, nn
from vllm.distributed import tensor_model_parallel_all_gather
from vllm.model_executor.layers.fused_moe import GateLinear
from vllm.model_executor.layers.linear import ReplicatedLinear
from vllm.model_executor.models.transformers.fx_utils import (
find_node,
@@ -21,14 +21,10 @@ from vllm.model_executor.models.transformers.fx_utils import (
peel,
trace,
)
from vllm.model_executor.models.transformers.utils import named_state
from vllm.model_executor.models.utils import maybe_prefix, sequence_parallel_chunk
def named_state(module: nn.Module) -> Iterator[tuple[str, torch.Tensor]]:
"""`module`'s own state (i.e. named parameters and buffers)."""
return chain(module.named_parameters(), module.named_buffers())
def _own_returns(node: ast.AST) -> Iterator[ast.Return]:
"""`return` statements in `node`'s own scope, not in nested functions."""
stack = list(ast.iter_child_nodes(node))
@@ -79,6 +75,25 @@ def _is_scalar_gate(module: nn.Module) -> bool:
)
def _forced_dtype(nodes: Iterable[fx.Node]) -> torch.dtype | None:
"""The floating dtype `nodes` cast to, if any.
Computations that must run in higher precision say so in their forward (e.g.
`hidden_states.type(torch.float32)`), so the dtype is readable from the graph
even when the config does not name it."""
for node in nodes:
if node.op not in ("call_method", "call_function"):
continue
name = str(node.target).rsplit(".", 1)[-1]
if name == "float":
return torch.float32
if name in ("to", "type"):
for arg in (*node.args[1:], *node.kwargs.values()):
if isinstance(arg, torch.dtype) and arg.is_floating_point:
return arg
return None
def _reaches(node: fx.Node, key: str) -> set[fx.Node]:
"""Returns the set of nodes reachable from `node` by following `key` edges."""
seen: set[fx.Node] = set()
@@ -131,18 +146,24 @@ class MoEBlockFuser:
scoring_func: str
shared_name: str | None
shared_gate_name: str | None
router_dtype: torch.dtype | None = None
@staticmethod
def _match_router(gate: nn.Module) -> str | None:
"""Matches `topk(score(linear(x)))`, `score` being `softmax`/`sigmoid`."""
if [name for name, _ in named_state(gate)] != ["weight"]:
def _match_router(gate: nn.Module) -> tuple[str, torch.dtype | None] | None:
"""Matches `topk(score(linear(x)))`, `score` being `softmax`/`sigmoid`.
Returns the scoring function and the dtype the router computes in."""
state = {name for name, _ in named_state(gate)}
if "weight" not in state or state - {"weight", "e_score_correction_bias"}:
return None
graph = trace(gate)
if graph is None:
return None
topk = find_node(graph, lambda n: is_op(n, "topk"))
if topk is None:
# The routing top-k is the last one; any earlier one scores expert groups.
topks = [node for node in graph.nodes if is_op(node, "topk")]
if not topks:
return None
topk = topks[-1]
# Exactly one scoring op upstream of the top-k, fed (transitively) by a linear.
scorers = [
n
@@ -152,9 +173,11 @@ class MoEBlockFuser:
if len(scorers) != 1:
return None
scorer = scorers[0]
if not any(is_op(n, "linear") for n in _reaches(scorer, "all_input_nodes")):
logits_cone = _reaches(scorer, "all_input_nodes")
if not any(is_op(n, "linear") for n in logits_cone):
return None
return "softmax" if is_op(scorer, "softmax") else "sigmoid"
scoring_func = "softmax" if is_op(scorer, "softmax") else "sigmoid"
return scoring_func, _forced_dtype(logits_cone)
@staticmethod
def _match_shared_experts(
@@ -198,10 +221,14 @@ class MoEBlockFuser:
if _returns_tuple(type(moe_block)):
return None
# Router: the child that scores + top-k selects.
gate_name = scoring_func = None
gate_name = scoring_func = router_dtype = None
for name, child in moe_block.named_children():
if name != experts_name and (func := cls._match_router(child)) is not None:
gate_name, scoring_func = name, func
if (
name != experts_name
and (router := cls._match_router(child)) is not None
):
gate_name = name
scoring_func, router_dtype = router
break
if gate_name is None or scoring_func is None:
return None
@@ -229,17 +256,23 @@ class MoEBlockFuser:
for name, child in moe_block.named_children():
if name not in accounted and next(named_state(child), None) is not None:
return None
return cls(gate_name, scoring_func, shared_name, shared_gate_name)
return cls(gate_name, scoring_func, shared_name, shared_gate_name, router_dtype)
def gate(self, moe_block: nn.Module, prefix: str) -> ReplicatedLinear:
"""Rebuild the HF gate as a `ReplicatedLinear` for vLLM's fused MoE."""
num_experts, hidden_size = getattr(moe_block, self.gate_name).weight.shape
gate = ReplicatedLinear(
def gate(
self, moe_block: nn.Module, prefix: str, out_dtype: torch.dtype | None = None
) -> GateLinear:
"""Rebuild the HF gate as a `GateLinear` for vLLM's fused MoE."""
hf_gate = getattr(moe_block, self.gate_name)
num_experts, hidden_size = hf_gate.weight.shape
gate = GateLinear(
hidden_size,
num_experts,
bias=False,
out_dtype=out_dtype or self.router_dtype,
prefix=maybe_prefix(prefix, self.gate_name),
)
if (bias := getattr(hf_gate, "e_score_correction_bias", None)) is not None:
gate.register_buffer("e_score_correction_bias", bias)
setattr(moe_block, self.gate_name, gate)
return gate
+73 -13
View File
@@ -23,6 +23,7 @@ from typing import TYPE_CHECKING, Any
import torch
import torch.nn as nn
from vllm._aiter_ops import rocm_aiter_ops
from vllm.config.utils import getattr_iter
from vllm.distributed import get_dp_group, get_ep_group
from vllm.forward_context import ForwardContext, get_forward_context
@@ -30,9 +31,10 @@ from vllm.logger import init_logger
from vllm.model_executor.custom_op import PluggableLayer
from vllm.model_executor.layers.fused_moe import FusedMoE, MoERunner, RoutedExperts
from vllm.model_executor.models.interfaces import MixtureOfExperts
from vllm.model_executor.models.transformers.fuser import get_fuser
from vllm.model_executor.models.transformers.fusers.moe import MoEBlockFuser
from vllm.model_executor.models.utils import maybe_prefix
from vllm.utils.torch_utils import direct_register_custom_op
from vllm.utils.torch_utils import STR_DTYPE_TO_TORCH_DTYPE, direct_register_custom_op
from .utils import log_replacement
@@ -176,17 +178,31 @@ class MoEMixin(MixtureOfExperts):
0,
)
# Unused kwargs since we use custom_routing_function:
# - `scoring_func` and `e_score_correction_bias` only used for grouped
# topk routing inside vLLM and are non-trivial to infer
# and hard code `use_grouped_topk=False`
# - `renormalize` passed anyway because it's easy to infer
# - `num_expert_group` and `topk_group` used for inferring expert
# placement strategy in FusedMoE
# - `apply_router_weight_on_input` is already applied in Transformers
# Common kwargs
renormalize = getattr(text_config, "norm_topk_prob", top_k > 1)
# Routed scaling factor kwargs
routed_scaling_factor = getattr(text_config, "routed_scaling_factor", 1.0)
# aiter applies routed_scaling_factor internally
apply_routed_scale_to_output = not rocm_aiter_ops.is_fused_moe_enabled()
routed_scaling_factor_kwargs = dict(
routed_scaling_factor=routed_scaling_factor,
apply_routed_scale_to_output=apply_routed_scale_to_output,
)
# Dtype the router computes in, if it is not the activation dtype.
config_router_dtype = getattr(text_config, "moe_router_dtype", None)
config_router_dtype = STR_DTYPE_TO_TORCH_DTYPE.get(config_router_dtype)
# Grouped topk routing kwargs
num_expert_group = getattr(text_config, "n_group", None)
topk_group = getattr(text_config, "topk_group", None)
use_grouped_topk = num_expert_group is not None and topk_group is not None
grouped_topk_routing_kwargs = dict(
use_grouped_topk=use_grouped_topk,
num_expert_group=num_expert_group,
topk_group=topk_group,
)
# MoE activation function
activation = "silu"
@@ -211,6 +227,9 @@ class MoEMixin(MixtureOfExperts):
self.num_shared_experts = num_shared_experts
self.num_redundant_experts = num_redundant_experts
# Down projections of shared experts consumed by FusedMoE
shared_down_projs: list[tuple[nn.Module, str]] = []
# Recursively fuse MoE layers
def _recursive_replace(module: nn.Module, prefix: str):
for child_name, child_module in module.named_children():
@@ -249,7 +268,6 @@ class MoEMixin(MixtureOfExperts):
hidden_size=hidden_size,
intermediate_size=intermediate_size,
renormalize=renormalize,
use_grouped_topk=False,
quant_config=self.quant_config,
prefix=qual_name,
activation=activation,
@@ -259,17 +277,55 @@ class MoEMixin(MixtureOfExperts):
routed_experts_cls=TransformersRoutedExperts,
)
fuser = MoEBlockFuser.match(moe_block, experts_name)
if self.num_expert_groups <= 1 and fuser is not None:
# _maybe_apply_routed_scale_to_output edge case. Transformers
# decoder layers do not compensate for dividing by scaling factor.
reaches_fp16_trick = (
routed_scaling_factor != 1.0
and apply_routed_scale_to_output
and self.model_config.dtype == torch.float16
and fuser is not None
and fuser.shared_name is not None
)
if reaches_fp16_trick:
logger.warning_once(
"%s could be fused but routing it in vLLM would apply "
"`routed_scaling_factor` by dividing the shared expert "
"output, which only fp16 overflow protection expects the "
"decoder layer to compensate for. Falling back to routing "
"in Transformers; run in bfloat16 to fuse it.",
moe_block_cls,
)
if fuser is not None and not reaches_fp16_trick:
# MoE block forward is fully replaced.
# gate/router and shared expert (if any) runs in FusedMoE.
shared_experts = fuser.shared_experts(moe_block, prefix)
# Store shared experts for later down projection adjustment
if shared_experts is not None:
hf_shared = shared_experts.shared_experts
glu_fuser = get_fuser(hf_shared)
down_name = getattr(glu_fuser, "down_name", None)
if down_name is not None:
shared_down_projs.append((hf_shared, down_name))
# Prefer config, otherwise read it from fuser.
router_dtype = config_router_dtype or fuser.router_dtype
gate = fuser.gate(moe_block, prefix, router_dtype)
kwargs |= dict(
scoring_func=fuser.scoring_func,
is_sequence_parallel=(
self.parallel_config.use_sequence_parallel_moe
),
gate=fuser.gate(moe_block, prefix),
shared_experts=fuser.shared_experts(moe_block, prefix),
gate=gate,
shared_experts=shared_experts,
)
if router_dtype is not None:
kwargs["router_logits_dtype"] = router_dtype
if use_grouped_topk:
kwargs |= grouped_topk_routing_kwargs
if routed_scaling_factor != 1.0:
kwargs |= routed_scaling_factor_kwargs
bias = getattr(gate, "e_score_correction_bias", None)
if bias is not None:
kwargs["e_score_correction_bias"] = bias
fuser.rewrite_forward(moe_block)
routed = "gate + experts"
if fuser.shared_name:
@@ -333,3 +389,7 @@ class MoEMixin(MixtureOfExperts):
self.num_moe_layers = len(self.moe_layers)
# Continue with the replacement of layers in Base
super().recursive_replace()
# GLUFuser likely fused shared_experts. The down projection after the GLU
# normally immediately reduces but we want FusedMoE to handle the reduction.
for hf_shared, down_name in shared_down_projs:
hf_shared.get_submodule(down_name).reduce_results = False
@@ -16,7 +16,9 @@
# limitations under the License.
"""Transformers modeling backend utilities."""
from collections.abc import Iterator
from contextlib import contextmanager
from itertools import chain
from pathlib import Path
from typing import TYPE_CHECKING, Literal
@@ -209,6 +211,11 @@ def recursive_replace_linear(
_recursive_replace(model, prefix=prefix)
def named_state(module: nn.Module) -> Iterator[tuple[str, torch.Tensor]]:
"""`module`'s own state (i.e. named parameters and buffers)."""
return chain(module.named_parameters(), module.named_buffers())
def log_replacement(name: str, old_module: nn.Module, new_module: nn.Module):
logger.debug("%s: %s -> %s", name, old_module, new_module)