Fix MLA padding and grouped topk routing in the Transformers modelling backend (#49982)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5
parent
bf2b45b5d6
commit
99de48e98f
@@ -29,8 +29,48 @@ class TopKRouter(nn.Module):
|
||||
return logits, value, index
|
||||
|
||||
|
||||
class ScaledRouter(TopKRouter):
|
||||
"""Greedy router scaling its top-k weights (DeepSeek `routed_scaling_factor`)."""
|
||||
|
||||
def forward(self, hidden_states):
|
||||
logits = F.linear(hidden_states, self.weight)
|
||||
scores = F.softmax(logits, dim=-1)
|
||||
value, index = torch.topk(scores, self.top_k, dim=-1)
|
||||
value = value * 16.0
|
||||
return logits, value, index
|
||||
|
||||
|
||||
class Fp32Router(TopKRouter):
|
||||
"""Router that computes its logits in fp32 (DeepSeek/GLM style)."""
|
||||
|
||||
def forward(self, hidden_states):
|
||||
logits = F.linear(
|
||||
hidden_states.type(torch.float32), self.weight.type(torch.float32)
|
||||
)
|
||||
scores = F.softmax(logits, dim=-1)
|
||||
value, index = torch.topk(scores, self.top_k, dim=-1)
|
||||
return logits, value.to(hidden_states.dtype), index
|
||||
|
||||
|
||||
class GroupedRouter(TopKRouter):
|
||||
"""Group-limited router (DeepSeek `group_limited_greedy`), scaled weights."""
|
||||
|
||||
def forward(self, hidden_states):
|
||||
logits = F.linear(hidden_states, self.weight)
|
||||
scores = F.softmax(logits, dim=-1)
|
||||
group_scores = scores.view(-1, 4, 2).max(dim=-1).values
|
||||
group_idx = torch.topk(group_scores, k=2, dim=-1)[1]
|
||||
group_mask = torch.zeros_like(group_scores)
|
||||
group_mask.scatter_(1, group_idx, 1)
|
||||
score_mask = group_mask.unsqueeze(-1).expand(-1, 4, 2).reshape(-1, 8)
|
||||
scores = scores.masked_fill(~score_mask.bool(), 0.0)
|
||||
value, index = torch.topk(scores, self.top_k, dim=-1)
|
||||
value = value * 16.0
|
||||
return logits, value, index
|
||||
|
||||
|
||||
class CorrectionRouter(nn.Module):
|
||||
"""Grouped router with a score-correction bias buffer (DeepSeek-V3) -> declined."""
|
||||
"""Router with a score-correction bias buffer (DeepSeek-V3 noaux) -> matched."""
|
||||
|
||||
def __init__(self, num_experts=8, hidden=16):
|
||||
super().__init__()
|
||||
@@ -228,6 +268,44 @@ def test_moe_fuser_detects_router(sigmoid):
|
||||
assert fuser.shared_name is None and fuser.shared_gate_name is None
|
||||
|
||||
|
||||
def test_moe_fuser_matches_scaled_router():
|
||||
"""Weight scaling after the top-k (DeepSeek style) does not break matching."""
|
||||
with torch.device("meta"):
|
||||
block = MoEBlock(ScaledRouter)
|
||||
assert isinstance(MoEBlockFuser.match(block, "experts"), MoEBlockFuser)
|
||||
|
||||
|
||||
def test_moe_fuser_matches_grouped_router():
|
||||
"""Group masking between the score and the routing top-k still matches: the
|
||||
scorer is anchored on the routing (last) top-k, not the group one."""
|
||||
with torch.device("meta"):
|
||||
block = MoEBlock(GroupedRouter)
|
||||
fuser = MoEBlockFuser.match(block, "experts")
|
||||
assert isinstance(fuser, MoEBlockFuser)
|
||||
assert fuser.scoring_func == "softmax"
|
||||
|
||||
|
||||
def test_moe_fuser_matches_correction_router():
|
||||
"""A score-correction bias buffer (DeepSeek-V3 noaux) is allowed; the
|
||||
rebuilt gate carries it in fp32 for FusedMoE's biased routers."""
|
||||
with torch.device("meta"):
|
||||
block = MoEBlock(CorrectionRouter)
|
||||
fuser = MoEBlockFuser.match(block, "experts")
|
||||
assert isinstance(fuser, MoEBlockFuser)
|
||||
assert fuser.scoring_func == "sigmoid"
|
||||
|
||||
|
||||
def test_moe_fuser_reads_router_dtype_from_the_gate():
|
||||
"""A router that computes its logits in fp32 must keep routing in fp32 when
|
||||
rebuilt, even though no config field names the dtype. The cast back to the
|
||||
activation dtype after the top-k must not be mistaken for the routing dtype."""
|
||||
with torch.device("meta"):
|
||||
fp32 = MoEBlockFuser.match(MoEBlock(Fp32Router), "experts")
|
||||
default = MoEBlockFuser.match(MoEBlock(TopKRouter), "experts")
|
||||
assert fp32.router_dtype == torch.float32
|
||||
assert default.router_dtype is None
|
||||
|
||||
|
||||
def test_moe_fuser_detects_shared_experts():
|
||||
with torch.device("meta"):
|
||||
block = MoEBlockShared()
|
||||
@@ -269,8 +347,7 @@ def test_moe_fuser_detects_non_glu_shared_expert():
|
||||
@pytest.mark.parametrize(
|
||||
"block_cls",
|
||||
[
|
||||
lambda: MoEBlock(CorrectionRouter), # score-correction buffer (grouped)
|
||||
lambda: MoEBlock(BiasedRouter), # router not weight-only (extra param)
|
||||
lambda: MoEBlock(BiasedRouter), # router with an unrecognized extra param
|
||||
MoEBlockTuple, # tuple-returning block (e.g. gpt-oss)
|
||||
MoEBlockTupleVar, # tuple returned via a name binding, not a literal
|
||||
MoEBlockUnaccounted, # weight-bearing child outside the fused dataflow
|
||||
|
||||
@@ -18,6 +18,7 @@
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import torch.nn.functional as F
|
||||
from transformers.modeling_utils import ALL_ATTENTION_FUNCTIONS
|
||||
|
||||
from vllm.model_executor.models.transformers.base import Base
|
||||
@@ -59,9 +60,19 @@ def vllm_attention_forward(
|
||||
if scaling is not None:
|
||||
self_attn.impl.scale = float(scaling)
|
||||
hidden = query.shape[-2]
|
||||
head_dim_qk = query.shape[-1]
|
||||
head_dim_v = value.shape[-1]
|
||||
query, key, value = (x.transpose(1, 2) for x in (query, key, value))
|
||||
query, key, value = (x.reshape(hidden, -1) for x in (query, key, value))
|
||||
return self_attn.forward(query, key, value), None
|
||||
# Pad `value` up to the query/key head size when they differ (expanded MLA).
|
||||
if head_dim_v != head_dim_qk:
|
||||
value = F.pad(value.view(-1, head_dim_v), (0, head_dim_qk - head_dim_v))
|
||||
value = value.reshape(hidden, -1)
|
||||
attn_output = self_attn.forward(query, key, value)
|
||||
if head_dim_v != head_dim_qk:
|
||||
attn_output = attn_output.view(-1, head_dim_qk)[..., :head_dim_v]
|
||||
attn_output = attn_output.reshape(hidden, -1)
|
||||
return attn_output, None
|
||||
|
||||
|
||||
ALL_ATTENTION_FUNCTIONS["vllm"] = vllm_attention_forward
|
||||
|
||||
@@ -6,14 +6,14 @@ import ast
|
||||
import inspect
|
||||
import textwrap
|
||||
import types
|
||||
from collections.abc import Iterator
|
||||
from collections.abc import Iterable, Iterator
|
||||
from dataclasses import dataclass
|
||||
from itertools import chain
|
||||
|
||||
import torch
|
||||
from torch import fx, nn
|
||||
|
||||
from vllm.distributed import tensor_model_parallel_all_gather
|
||||
from vllm.model_executor.layers.fused_moe import GateLinear
|
||||
from vllm.model_executor.layers.linear import ReplicatedLinear
|
||||
from vllm.model_executor.models.transformers.fx_utils import (
|
||||
find_node,
|
||||
@@ -21,14 +21,10 @@ from vllm.model_executor.models.transformers.fx_utils import (
|
||||
peel,
|
||||
trace,
|
||||
)
|
||||
from vllm.model_executor.models.transformers.utils import named_state
|
||||
from vllm.model_executor.models.utils import maybe_prefix, sequence_parallel_chunk
|
||||
|
||||
|
||||
def named_state(module: nn.Module) -> Iterator[tuple[str, torch.Tensor]]:
|
||||
"""`module`'s own state (i.e. named parameters and buffers)."""
|
||||
return chain(module.named_parameters(), module.named_buffers())
|
||||
|
||||
|
||||
def _own_returns(node: ast.AST) -> Iterator[ast.Return]:
|
||||
"""`return` statements in `node`'s own scope, not in nested functions."""
|
||||
stack = list(ast.iter_child_nodes(node))
|
||||
@@ -79,6 +75,25 @@ def _is_scalar_gate(module: nn.Module) -> bool:
|
||||
)
|
||||
|
||||
|
||||
def _forced_dtype(nodes: Iterable[fx.Node]) -> torch.dtype | None:
|
||||
"""The floating dtype `nodes` cast to, if any.
|
||||
|
||||
Computations that must run in higher precision say so in their forward (e.g.
|
||||
`hidden_states.type(torch.float32)`), so the dtype is readable from the graph
|
||||
even when the config does not name it."""
|
||||
for node in nodes:
|
||||
if node.op not in ("call_method", "call_function"):
|
||||
continue
|
||||
name = str(node.target).rsplit(".", 1)[-1]
|
||||
if name == "float":
|
||||
return torch.float32
|
||||
if name in ("to", "type"):
|
||||
for arg in (*node.args[1:], *node.kwargs.values()):
|
||||
if isinstance(arg, torch.dtype) and arg.is_floating_point:
|
||||
return arg
|
||||
return None
|
||||
|
||||
|
||||
def _reaches(node: fx.Node, key: str) -> set[fx.Node]:
|
||||
"""Returns the set of nodes reachable from `node` by following `key` edges."""
|
||||
seen: set[fx.Node] = set()
|
||||
@@ -131,18 +146,24 @@ class MoEBlockFuser:
|
||||
scoring_func: str
|
||||
shared_name: str | None
|
||||
shared_gate_name: str | None
|
||||
router_dtype: torch.dtype | None = None
|
||||
|
||||
@staticmethod
|
||||
def _match_router(gate: nn.Module) -> str | None:
|
||||
"""Matches `topk(score(linear(x)))`, `score` being `softmax`/`sigmoid`."""
|
||||
if [name for name, _ in named_state(gate)] != ["weight"]:
|
||||
def _match_router(gate: nn.Module) -> tuple[str, torch.dtype | None] | None:
|
||||
"""Matches `topk(score(linear(x)))`, `score` being `softmax`/`sigmoid`.
|
||||
|
||||
Returns the scoring function and the dtype the router computes in."""
|
||||
state = {name for name, _ in named_state(gate)}
|
||||
if "weight" not in state or state - {"weight", "e_score_correction_bias"}:
|
||||
return None
|
||||
graph = trace(gate)
|
||||
if graph is None:
|
||||
return None
|
||||
topk = find_node(graph, lambda n: is_op(n, "topk"))
|
||||
if topk is None:
|
||||
# The routing top-k is the last one; any earlier one scores expert groups.
|
||||
topks = [node for node in graph.nodes if is_op(node, "topk")]
|
||||
if not topks:
|
||||
return None
|
||||
topk = topks[-1]
|
||||
# Exactly one scoring op upstream of the top-k, fed (transitively) by a linear.
|
||||
scorers = [
|
||||
n
|
||||
@@ -152,9 +173,11 @@ class MoEBlockFuser:
|
||||
if len(scorers) != 1:
|
||||
return None
|
||||
scorer = scorers[0]
|
||||
if not any(is_op(n, "linear") for n in _reaches(scorer, "all_input_nodes")):
|
||||
logits_cone = _reaches(scorer, "all_input_nodes")
|
||||
if not any(is_op(n, "linear") for n in logits_cone):
|
||||
return None
|
||||
return "softmax" if is_op(scorer, "softmax") else "sigmoid"
|
||||
scoring_func = "softmax" if is_op(scorer, "softmax") else "sigmoid"
|
||||
return scoring_func, _forced_dtype(logits_cone)
|
||||
|
||||
@staticmethod
|
||||
def _match_shared_experts(
|
||||
@@ -198,10 +221,14 @@ class MoEBlockFuser:
|
||||
if _returns_tuple(type(moe_block)):
|
||||
return None
|
||||
# Router: the child that scores + top-k selects.
|
||||
gate_name = scoring_func = None
|
||||
gate_name = scoring_func = router_dtype = None
|
||||
for name, child in moe_block.named_children():
|
||||
if name != experts_name and (func := cls._match_router(child)) is not None:
|
||||
gate_name, scoring_func = name, func
|
||||
if (
|
||||
name != experts_name
|
||||
and (router := cls._match_router(child)) is not None
|
||||
):
|
||||
gate_name = name
|
||||
scoring_func, router_dtype = router
|
||||
break
|
||||
if gate_name is None or scoring_func is None:
|
||||
return None
|
||||
@@ -229,17 +256,23 @@ class MoEBlockFuser:
|
||||
for name, child in moe_block.named_children():
|
||||
if name not in accounted and next(named_state(child), None) is not None:
|
||||
return None
|
||||
return cls(gate_name, scoring_func, shared_name, shared_gate_name)
|
||||
return cls(gate_name, scoring_func, shared_name, shared_gate_name, router_dtype)
|
||||
|
||||
def gate(self, moe_block: nn.Module, prefix: str) -> ReplicatedLinear:
|
||||
"""Rebuild the HF gate as a `ReplicatedLinear` for vLLM's fused MoE."""
|
||||
num_experts, hidden_size = getattr(moe_block, self.gate_name).weight.shape
|
||||
gate = ReplicatedLinear(
|
||||
def gate(
|
||||
self, moe_block: nn.Module, prefix: str, out_dtype: torch.dtype | None = None
|
||||
) -> GateLinear:
|
||||
"""Rebuild the HF gate as a `GateLinear` for vLLM's fused MoE."""
|
||||
hf_gate = getattr(moe_block, self.gate_name)
|
||||
num_experts, hidden_size = hf_gate.weight.shape
|
||||
gate = GateLinear(
|
||||
hidden_size,
|
||||
num_experts,
|
||||
bias=False,
|
||||
out_dtype=out_dtype or self.router_dtype,
|
||||
prefix=maybe_prefix(prefix, self.gate_name),
|
||||
)
|
||||
if (bias := getattr(hf_gate, "e_score_correction_bias", None)) is not None:
|
||||
gate.register_buffer("e_score_correction_bias", bias)
|
||||
setattr(moe_block, self.gate_name, gate)
|
||||
return gate
|
||||
|
||||
|
||||
@@ -23,6 +23,7 @@ from typing import TYPE_CHECKING, Any
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.config.utils import getattr_iter
|
||||
from vllm.distributed import get_dp_group, get_ep_group
|
||||
from vllm.forward_context import ForwardContext, get_forward_context
|
||||
@@ -30,9 +31,10 @@ from vllm.logger import init_logger
|
||||
from vllm.model_executor.custom_op import PluggableLayer
|
||||
from vllm.model_executor.layers.fused_moe import FusedMoE, MoERunner, RoutedExperts
|
||||
from vllm.model_executor.models.interfaces import MixtureOfExperts
|
||||
from vllm.model_executor.models.transformers.fuser import get_fuser
|
||||
from vllm.model_executor.models.transformers.fusers.moe import MoEBlockFuser
|
||||
from vllm.model_executor.models.utils import maybe_prefix
|
||||
from vllm.utils.torch_utils import direct_register_custom_op
|
||||
from vllm.utils.torch_utils import STR_DTYPE_TO_TORCH_DTYPE, direct_register_custom_op
|
||||
|
||||
from .utils import log_replacement
|
||||
|
||||
@@ -176,17 +178,31 @@ class MoEMixin(MixtureOfExperts):
|
||||
0,
|
||||
)
|
||||
|
||||
# Unused kwargs since we use custom_routing_function:
|
||||
# - `scoring_func` and `e_score_correction_bias` only used for grouped
|
||||
# topk routing inside vLLM and are non-trivial to infer
|
||||
# and hard code `use_grouped_topk=False`
|
||||
# - `renormalize` passed anyway because it's easy to infer
|
||||
# - `num_expert_group` and `topk_group` used for inferring expert
|
||||
# placement strategy in FusedMoE
|
||||
# - `apply_router_weight_on_input` is already applied in Transformers
|
||||
# Common kwargs
|
||||
renormalize = getattr(text_config, "norm_topk_prob", top_k > 1)
|
||||
|
||||
# Routed scaling factor kwargs
|
||||
routed_scaling_factor = getattr(text_config, "routed_scaling_factor", 1.0)
|
||||
# aiter applies routed_scaling_factor internally
|
||||
apply_routed_scale_to_output = not rocm_aiter_ops.is_fused_moe_enabled()
|
||||
routed_scaling_factor_kwargs = dict(
|
||||
routed_scaling_factor=routed_scaling_factor,
|
||||
apply_routed_scale_to_output=apply_routed_scale_to_output,
|
||||
)
|
||||
|
||||
# Dtype the router computes in, if it is not the activation dtype.
|
||||
config_router_dtype = getattr(text_config, "moe_router_dtype", None)
|
||||
config_router_dtype = STR_DTYPE_TO_TORCH_DTYPE.get(config_router_dtype)
|
||||
|
||||
# Grouped topk routing kwargs
|
||||
num_expert_group = getattr(text_config, "n_group", None)
|
||||
topk_group = getattr(text_config, "topk_group", None)
|
||||
use_grouped_topk = num_expert_group is not None and topk_group is not None
|
||||
grouped_topk_routing_kwargs = dict(
|
||||
use_grouped_topk=use_grouped_topk,
|
||||
num_expert_group=num_expert_group,
|
||||
topk_group=topk_group,
|
||||
)
|
||||
|
||||
# MoE activation function
|
||||
activation = "silu"
|
||||
@@ -211,6 +227,9 @@ class MoEMixin(MixtureOfExperts):
|
||||
self.num_shared_experts = num_shared_experts
|
||||
self.num_redundant_experts = num_redundant_experts
|
||||
|
||||
# Down projections of shared experts consumed by FusedMoE
|
||||
shared_down_projs: list[tuple[nn.Module, str]] = []
|
||||
|
||||
# Recursively fuse MoE layers
|
||||
def _recursive_replace(module: nn.Module, prefix: str):
|
||||
for child_name, child_module in module.named_children():
|
||||
@@ -249,7 +268,6 @@ class MoEMixin(MixtureOfExperts):
|
||||
hidden_size=hidden_size,
|
||||
intermediate_size=intermediate_size,
|
||||
renormalize=renormalize,
|
||||
use_grouped_topk=False,
|
||||
quant_config=self.quant_config,
|
||||
prefix=qual_name,
|
||||
activation=activation,
|
||||
@@ -259,17 +277,55 @@ class MoEMixin(MixtureOfExperts):
|
||||
routed_experts_cls=TransformersRoutedExperts,
|
||||
)
|
||||
fuser = MoEBlockFuser.match(moe_block, experts_name)
|
||||
if self.num_expert_groups <= 1 and fuser is not None:
|
||||
# _maybe_apply_routed_scale_to_output edge case. Transformers
|
||||
# decoder layers do not compensate for dividing by scaling factor.
|
||||
reaches_fp16_trick = (
|
||||
routed_scaling_factor != 1.0
|
||||
and apply_routed_scale_to_output
|
||||
and self.model_config.dtype == torch.float16
|
||||
and fuser is not None
|
||||
and fuser.shared_name is not None
|
||||
)
|
||||
if reaches_fp16_trick:
|
||||
logger.warning_once(
|
||||
"%s could be fused but routing it in vLLM would apply "
|
||||
"`routed_scaling_factor` by dividing the shared expert "
|
||||
"output, which only fp16 overflow protection expects the "
|
||||
"decoder layer to compensate for. Falling back to routing "
|
||||
"in Transformers; run in bfloat16 to fuse it.",
|
||||
moe_block_cls,
|
||||
)
|
||||
if fuser is not None and not reaches_fp16_trick:
|
||||
# MoE block forward is fully replaced.
|
||||
# gate/router and shared expert (if any) runs in FusedMoE.
|
||||
shared_experts = fuser.shared_experts(moe_block, prefix)
|
||||
# Store shared experts for later down projection adjustment
|
||||
if shared_experts is not None:
|
||||
hf_shared = shared_experts.shared_experts
|
||||
glu_fuser = get_fuser(hf_shared)
|
||||
down_name = getattr(glu_fuser, "down_name", None)
|
||||
if down_name is not None:
|
||||
shared_down_projs.append((hf_shared, down_name))
|
||||
# Prefer config, otherwise read it from fuser.
|
||||
router_dtype = config_router_dtype or fuser.router_dtype
|
||||
gate = fuser.gate(moe_block, prefix, router_dtype)
|
||||
kwargs |= dict(
|
||||
scoring_func=fuser.scoring_func,
|
||||
is_sequence_parallel=(
|
||||
self.parallel_config.use_sequence_parallel_moe
|
||||
),
|
||||
gate=fuser.gate(moe_block, prefix),
|
||||
shared_experts=fuser.shared_experts(moe_block, prefix),
|
||||
gate=gate,
|
||||
shared_experts=shared_experts,
|
||||
)
|
||||
if router_dtype is not None:
|
||||
kwargs["router_logits_dtype"] = router_dtype
|
||||
if use_grouped_topk:
|
||||
kwargs |= grouped_topk_routing_kwargs
|
||||
if routed_scaling_factor != 1.0:
|
||||
kwargs |= routed_scaling_factor_kwargs
|
||||
bias = getattr(gate, "e_score_correction_bias", None)
|
||||
if bias is not None:
|
||||
kwargs["e_score_correction_bias"] = bias
|
||||
fuser.rewrite_forward(moe_block)
|
||||
routed = "gate + experts"
|
||||
if fuser.shared_name:
|
||||
@@ -333,3 +389,7 @@ class MoEMixin(MixtureOfExperts):
|
||||
self.num_moe_layers = len(self.moe_layers)
|
||||
# Continue with the replacement of layers in Base
|
||||
super().recursive_replace()
|
||||
# GLUFuser likely fused shared_experts. The down projection after the GLU
|
||||
# normally immediately reduces but we want FusedMoE to handle the reduction.
|
||||
for hf_shared, down_name in shared_down_projs:
|
||||
hf_shared.get_submodule(down_name).reduce_results = False
|
||||
|
||||
@@ -16,7 +16,9 @@
|
||||
# limitations under the License.
|
||||
"""Transformers modeling backend utilities."""
|
||||
|
||||
from collections.abc import Iterator
|
||||
from contextlib import contextmanager
|
||||
from itertools import chain
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING, Literal
|
||||
|
||||
@@ -209,6 +211,11 @@ def recursive_replace_linear(
|
||||
_recursive_replace(model, prefix=prefix)
|
||||
|
||||
|
||||
def named_state(module: nn.Module) -> Iterator[tuple[str, torch.Tensor]]:
|
||||
"""`module`'s own state (i.e. named parameters and buffers)."""
|
||||
return chain(module.named_parameters(), module.named_buffers())
|
||||
|
||||
|
||||
def log_replacement(name: str, old_module: nn.Module, new_module: nn.Module):
|
||||
logger.debug("%s: %s -> %s", name, old_module, new_module)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user