diff --git a/tests/models/transformers/fusers/test_moe.py b/tests/models/transformers/fusers/test_moe.py index 04eadac3f7..7de4589eef 100644 --- a/tests/models/transformers/fusers/test_moe.py +++ b/tests/models/transformers/fusers/test_moe.py @@ -29,8 +29,48 @@ class TopKRouter(nn.Module): return logits, value, index +class ScaledRouter(TopKRouter): + """Greedy router scaling its top-k weights (DeepSeek `routed_scaling_factor`).""" + + def forward(self, hidden_states): + logits = F.linear(hidden_states, self.weight) + scores = F.softmax(logits, dim=-1) + value, index = torch.topk(scores, self.top_k, dim=-1) + value = value * 16.0 + return logits, value, index + + +class Fp32Router(TopKRouter): + """Router that computes its logits in fp32 (DeepSeek/GLM style).""" + + def forward(self, hidden_states): + logits = F.linear( + hidden_states.type(torch.float32), self.weight.type(torch.float32) + ) + scores = F.softmax(logits, dim=-1) + value, index = torch.topk(scores, self.top_k, dim=-1) + return logits, value.to(hidden_states.dtype), index + + +class GroupedRouter(TopKRouter): + """Group-limited router (DeepSeek `group_limited_greedy`), scaled weights.""" + + def forward(self, hidden_states): + logits = F.linear(hidden_states, self.weight) + scores = F.softmax(logits, dim=-1) + group_scores = scores.view(-1, 4, 2).max(dim=-1).values + group_idx = torch.topk(group_scores, k=2, dim=-1)[1] + group_mask = torch.zeros_like(group_scores) + group_mask.scatter_(1, group_idx, 1) + score_mask = group_mask.unsqueeze(-1).expand(-1, 4, 2).reshape(-1, 8) + scores = scores.masked_fill(~score_mask.bool(), 0.0) + value, index = torch.topk(scores, self.top_k, dim=-1) + value = value * 16.0 + return logits, value, index + + class CorrectionRouter(nn.Module): - """Grouped router with a score-correction bias buffer (DeepSeek-V3) -> declined.""" + """Router with a score-correction bias buffer (DeepSeek-V3 noaux) -> matched.""" def __init__(self, num_experts=8, hidden=16): super().__init__() @@ -228,6 +268,44 @@ def test_moe_fuser_detects_router(sigmoid): assert fuser.shared_name is None and fuser.shared_gate_name is None +def test_moe_fuser_matches_scaled_router(): + """Weight scaling after the top-k (DeepSeek style) does not break matching.""" + with torch.device("meta"): + block = MoEBlock(ScaledRouter) + assert isinstance(MoEBlockFuser.match(block, "experts"), MoEBlockFuser) + + +def test_moe_fuser_matches_grouped_router(): + """Group masking between the score and the routing top-k still matches: the + scorer is anchored on the routing (last) top-k, not the group one.""" + with torch.device("meta"): + block = MoEBlock(GroupedRouter) + fuser = MoEBlockFuser.match(block, "experts") + assert isinstance(fuser, MoEBlockFuser) + assert fuser.scoring_func == "softmax" + + +def test_moe_fuser_matches_correction_router(): + """A score-correction bias buffer (DeepSeek-V3 noaux) is allowed; the + rebuilt gate carries it in fp32 for FusedMoE's biased routers.""" + with torch.device("meta"): + block = MoEBlock(CorrectionRouter) + fuser = MoEBlockFuser.match(block, "experts") + assert isinstance(fuser, MoEBlockFuser) + assert fuser.scoring_func == "sigmoid" + + +def test_moe_fuser_reads_router_dtype_from_the_gate(): + """A router that computes its logits in fp32 must keep routing in fp32 when + rebuilt, even though no config field names the dtype. The cast back to the + activation dtype after the top-k must not be mistaken for the routing dtype.""" + with torch.device("meta"): + fp32 = MoEBlockFuser.match(MoEBlock(Fp32Router), "experts") + default = MoEBlockFuser.match(MoEBlock(TopKRouter), "experts") + assert fp32.router_dtype == torch.float32 + assert default.router_dtype is None + + def test_moe_fuser_detects_shared_experts(): with torch.device("meta"): block = MoEBlockShared() @@ -269,8 +347,7 @@ def test_moe_fuser_detects_non_glu_shared_expert(): @pytest.mark.parametrize( "block_cls", [ - lambda: MoEBlock(CorrectionRouter), # score-correction buffer (grouped) - lambda: MoEBlock(BiasedRouter), # router not weight-only (extra param) + lambda: MoEBlock(BiasedRouter), # router with an unrecognized extra param MoEBlockTuple, # tuple-returning block (e.g. gpt-oss) MoEBlockTupleVar, # tuple returned via a name binding, not a literal MoEBlockUnaccounted, # weight-bearing child outside the fused dataflow diff --git a/vllm/model_executor/models/transformers/__init__.py b/vllm/model_executor/models/transformers/__init__.py index 78a12876e6..9dbfe4b503 100644 --- a/vllm/model_executor/models/transformers/__init__.py +++ b/vllm/model_executor/models/transformers/__init__.py @@ -18,6 +18,7 @@ from typing import TYPE_CHECKING +import torch.nn.functional as F from transformers.modeling_utils import ALL_ATTENTION_FUNCTIONS from vllm.model_executor.models.transformers.base import Base @@ -59,9 +60,19 @@ def vllm_attention_forward( if scaling is not None: self_attn.impl.scale = float(scaling) hidden = query.shape[-2] + head_dim_qk = query.shape[-1] + head_dim_v = value.shape[-1] query, key, value = (x.transpose(1, 2) for x in (query, key, value)) query, key, value = (x.reshape(hidden, -1) for x in (query, key, value)) - return self_attn.forward(query, key, value), None + # Pad `value` up to the query/key head size when they differ (expanded MLA). + if head_dim_v != head_dim_qk: + value = F.pad(value.view(-1, head_dim_v), (0, head_dim_qk - head_dim_v)) + value = value.reshape(hidden, -1) + attn_output = self_attn.forward(query, key, value) + if head_dim_v != head_dim_qk: + attn_output = attn_output.view(-1, head_dim_qk)[..., :head_dim_v] + attn_output = attn_output.reshape(hidden, -1) + return attn_output, None ALL_ATTENTION_FUNCTIONS["vllm"] = vllm_attention_forward diff --git a/vllm/model_executor/models/transformers/fusers/moe.py b/vllm/model_executor/models/transformers/fusers/moe.py index 6a3c7e85d6..af3a101a0c 100644 --- a/vllm/model_executor/models/transformers/fusers/moe.py +++ b/vllm/model_executor/models/transformers/fusers/moe.py @@ -6,14 +6,14 @@ import ast import inspect import textwrap import types -from collections.abc import Iterator +from collections.abc import Iterable, Iterator from dataclasses import dataclass -from itertools import chain import torch from torch import fx, nn from vllm.distributed import tensor_model_parallel_all_gather +from vllm.model_executor.layers.fused_moe import GateLinear from vllm.model_executor.layers.linear import ReplicatedLinear from vllm.model_executor.models.transformers.fx_utils import ( find_node, @@ -21,14 +21,10 @@ from vllm.model_executor.models.transformers.fx_utils import ( peel, trace, ) +from vllm.model_executor.models.transformers.utils import named_state from vllm.model_executor.models.utils import maybe_prefix, sequence_parallel_chunk -def named_state(module: nn.Module) -> Iterator[tuple[str, torch.Tensor]]: - """`module`'s own state (i.e. named parameters and buffers).""" - return chain(module.named_parameters(), module.named_buffers()) - - def _own_returns(node: ast.AST) -> Iterator[ast.Return]: """`return` statements in `node`'s own scope, not in nested functions.""" stack = list(ast.iter_child_nodes(node)) @@ -79,6 +75,25 @@ def _is_scalar_gate(module: nn.Module) -> bool: ) +def _forced_dtype(nodes: Iterable[fx.Node]) -> torch.dtype | None: + """The floating dtype `nodes` cast to, if any. + + Computations that must run in higher precision say so in their forward (e.g. + `hidden_states.type(torch.float32)`), so the dtype is readable from the graph + even when the config does not name it.""" + for node in nodes: + if node.op not in ("call_method", "call_function"): + continue + name = str(node.target).rsplit(".", 1)[-1] + if name == "float": + return torch.float32 + if name in ("to", "type"): + for arg in (*node.args[1:], *node.kwargs.values()): + if isinstance(arg, torch.dtype) and arg.is_floating_point: + return arg + return None + + def _reaches(node: fx.Node, key: str) -> set[fx.Node]: """Returns the set of nodes reachable from `node` by following `key` edges.""" seen: set[fx.Node] = set() @@ -131,18 +146,24 @@ class MoEBlockFuser: scoring_func: str shared_name: str | None shared_gate_name: str | None + router_dtype: torch.dtype | None = None @staticmethod - def _match_router(gate: nn.Module) -> str | None: - """Matches `topk(score(linear(x)))`, `score` being `softmax`/`sigmoid`.""" - if [name for name, _ in named_state(gate)] != ["weight"]: + def _match_router(gate: nn.Module) -> tuple[str, torch.dtype | None] | None: + """Matches `topk(score(linear(x)))`, `score` being `softmax`/`sigmoid`. + + Returns the scoring function and the dtype the router computes in.""" + state = {name for name, _ in named_state(gate)} + if "weight" not in state or state - {"weight", "e_score_correction_bias"}: return None graph = trace(gate) if graph is None: return None - topk = find_node(graph, lambda n: is_op(n, "topk")) - if topk is None: + # The routing top-k is the last one; any earlier one scores expert groups. + topks = [node for node in graph.nodes if is_op(node, "topk")] + if not topks: return None + topk = topks[-1] # Exactly one scoring op upstream of the top-k, fed (transitively) by a linear. scorers = [ n @@ -152,9 +173,11 @@ class MoEBlockFuser: if len(scorers) != 1: return None scorer = scorers[0] - if not any(is_op(n, "linear") for n in _reaches(scorer, "all_input_nodes")): + logits_cone = _reaches(scorer, "all_input_nodes") + if not any(is_op(n, "linear") for n in logits_cone): return None - return "softmax" if is_op(scorer, "softmax") else "sigmoid" + scoring_func = "softmax" if is_op(scorer, "softmax") else "sigmoid" + return scoring_func, _forced_dtype(logits_cone) @staticmethod def _match_shared_experts( @@ -198,10 +221,14 @@ class MoEBlockFuser: if _returns_tuple(type(moe_block)): return None # Router: the child that scores + top-k selects. - gate_name = scoring_func = None + gate_name = scoring_func = router_dtype = None for name, child in moe_block.named_children(): - if name != experts_name and (func := cls._match_router(child)) is not None: - gate_name, scoring_func = name, func + if ( + name != experts_name + and (router := cls._match_router(child)) is not None + ): + gate_name = name + scoring_func, router_dtype = router break if gate_name is None or scoring_func is None: return None @@ -229,17 +256,23 @@ class MoEBlockFuser: for name, child in moe_block.named_children(): if name not in accounted and next(named_state(child), None) is not None: return None - return cls(gate_name, scoring_func, shared_name, shared_gate_name) + return cls(gate_name, scoring_func, shared_name, shared_gate_name, router_dtype) - def gate(self, moe_block: nn.Module, prefix: str) -> ReplicatedLinear: - """Rebuild the HF gate as a `ReplicatedLinear` for vLLM's fused MoE.""" - num_experts, hidden_size = getattr(moe_block, self.gate_name).weight.shape - gate = ReplicatedLinear( + def gate( + self, moe_block: nn.Module, prefix: str, out_dtype: torch.dtype | None = None + ) -> GateLinear: + """Rebuild the HF gate as a `GateLinear` for vLLM's fused MoE.""" + hf_gate = getattr(moe_block, self.gate_name) + num_experts, hidden_size = hf_gate.weight.shape + gate = GateLinear( hidden_size, num_experts, bias=False, + out_dtype=out_dtype or self.router_dtype, prefix=maybe_prefix(prefix, self.gate_name), ) + if (bias := getattr(hf_gate, "e_score_correction_bias", None)) is not None: + gate.register_buffer("e_score_correction_bias", bias) setattr(moe_block, self.gate_name, gate) return gate diff --git a/vllm/model_executor/models/transformers/moe.py b/vllm/model_executor/models/transformers/moe.py index d1f5dba037..ce96403697 100644 --- a/vllm/model_executor/models/transformers/moe.py +++ b/vllm/model_executor/models/transformers/moe.py @@ -23,6 +23,7 @@ from typing import TYPE_CHECKING, Any import torch import torch.nn as nn +from vllm._aiter_ops import rocm_aiter_ops from vllm.config.utils import getattr_iter from vllm.distributed import get_dp_group, get_ep_group from vllm.forward_context import ForwardContext, get_forward_context @@ -30,9 +31,10 @@ from vllm.logger import init_logger from vllm.model_executor.custom_op import PluggableLayer from vllm.model_executor.layers.fused_moe import FusedMoE, MoERunner, RoutedExperts from vllm.model_executor.models.interfaces import MixtureOfExperts +from vllm.model_executor.models.transformers.fuser import get_fuser from vllm.model_executor.models.transformers.fusers.moe import MoEBlockFuser from vllm.model_executor.models.utils import maybe_prefix -from vllm.utils.torch_utils import direct_register_custom_op +from vllm.utils.torch_utils import STR_DTYPE_TO_TORCH_DTYPE, direct_register_custom_op from .utils import log_replacement @@ -176,17 +178,31 @@ class MoEMixin(MixtureOfExperts): 0, ) - # Unused kwargs since we use custom_routing_function: - # - `scoring_func` and `e_score_correction_bias` only used for grouped - # topk routing inside vLLM and are non-trivial to infer - # and hard code `use_grouped_topk=False` - # - `renormalize` passed anyway because it's easy to infer - # - `num_expert_group` and `topk_group` used for inferring expert - # placement strategy in FusedMoE - # - `apply_router_weight_on_input` is already applied in Transformers + # Common kwargs renormalize = getattr(text_config, "norm_topk_prob", top_k > 1) + + # Routed scaling factor kwargs + routed_scaling_factor = getattr(text_config, "routed_scaling_factor", 1.0) + # aiter applies routed_scaling_factor internally + apply_routed_scale_to_output = not rocm_aiter_ops.is_fused_moe_enabled() + routed_scaling_factor_kwargs = dict( + routed_scaling_factor=routed_scaling_factor, + apply_routed_scale_to_output=apply_routed_scale_to_output, + ) + + # Dtype the router computes in, if it is not the activation dtype. + config_router_dtype = getattr(text_config, "moe_router_dtype", None) + config_router_dtype = STR_DTYPE_TO_TORCH_DTYPE.get(config_router_dtype) + + # Grouped topk routing kwargs num_expert_group = getattr(text_config, "n_group", None) topk_group = getattr(text_config, "topk_group", None) + use_grouped_topk = num_expert_group is not None and topk_group is not None + grouped_topk_routing_kwargs = dict( + use_grouped_topk=use_grouped_topk, + num_expert_group=num_expert_group, + topk_group=topk_group, + ) # MoE activation function activation = "silu" @@ -211,6 +227,9 @@ class MoEMixin(MixtureOfExperts): self.num_shared_experts = num_shared_experts self.num_redundant_experts = num_redundant_experts + # Down projections of shared experts consumed by FusedMoE + shared_down_projs: list[tuple[nn.Module, str]] = [] + # Recursively fuse MoE layers def _recursive_replace(module: nn.Module, prefix: str): for child_name, child_module in module.named_children(): @@ -249,7 +268,6 @@ class MoEMixin(MixtureOfExperts): hidden_size=hidden_size, intermediate_size=intermediate_size, renormalize=renormalize, - use_grouped_topk=False, quant_config=self.quant_config, prefix=qual_name, activation=activation, @@ -259,17 +277,55 @@ class MoEMixin(MixtureOfExperts): routed_experts_cls=TransformersRoutedExperts, ) fuser = MoEBlockFuser.match(moe_block, experts_name) - if self.num_expert_groups <= 1 and fuser is not None: + # _maybe_apply_routed_scale_to_output edge case. Transformers + # decoder layers do not compensate for dividing by scaling factor. + reaches_fp16_trick = ( + routed_scaling_factor != 1.0 + and apply_routed_scale_to_output + and self.model_config.dtype == torch.float16 + and fuser is not None + and fuser.shared_name is not None + ) + if reaches_fp16_trick: + logger.warning_once( + "%s could be fused but routing it in vLLM would apply " + "`routed_scaling_factor` by dividing the shared expert " + "output, which only fp16 overflow protection expects the " + "decoder layer to compensate for. Falling back to routing " + "in Transformers; run in bfloat16 to fuse it.", + moe_block_cls, + ) + if fuser is not None and not reaches_fp16_trick: # MoE block forward is fully replaced. # gate/router and shared expert (if any) runs in FusedMoE. + shared_experts = fuser.shared_experts(moe_block, prefix) + # Store shared experts for later down projection adjustment + if shared_experts is not None: + hf_shared = shared_experts.shared_experts + glu_fuser = get_fuser(hf_shared) + down_name = getattr(glu_fuser, "down_name", None) + if down_name is not None: + shared_down_projs.append((hf_shared, down_name)) + # Prefer config, otherwise read it from fuser. + router_dtype = config_router_dtype or fuser.router_dtype + gate = fuser.gate(moe_block, prefix, router_dtype) kwargs |= dict( scoring_func=fuser.scoring_func, is_sequence_parallel=( self.parallel_config.use_sequence_parallel_moe ), - gate=fuser.gate(moe_block, prefix), - shared_experts=fuser.shared_experts(moe_block, prefix), + gate=gate, + shared_experts=shared_experts, ) + if router_dtype is not None: + kwargs["router_logits_dtype"] = router_dtype + if use_grouped_topk: + kwargs |= grouped_topk_routing_kwargs + if routed_scaling_factor != 1.0: + kwargs |= routed_scaling_factor_kwargs + bias = getattr(gate, "e_score_correction_bias", None) + if bias is not None: + kwargs["e_score_correction_bias"] = bias fuser.rewrite_forward(moe_block) routed = "gate + experts" if fuser.shared_name: @@ -333,3 +389,7 @@ class MoEMixin(MixtureOfExperts): self.num_moe_layers = len(self.moe_layers) # Continue with the replacement of layers in Base super().recursive_replace() + # GLUFuser likely fused shared_experts. The down projection after the GLU + # normally immediately reduces but we want FusedMoE to handle the reduction. + for hf_shared, down_name in shared_down_projs: + hf_shared.get_submodule(down_name).reduce_results = False diff --git a/vllm/model_executor/models/transformers/utils.py b/vllm/model_executor/models/transformers/utils.py index 4d9b01ce39..9000ace905 100644 --- a/vllm/model_executor/models/transformers/utils.py +++ b/vllm/model_executor/models/transformers/utils.py @@ -16,7 +16,9 @@ # limitations under the License. """Transformers modeling backend utilities.""" +from collections.abc import Iterator from contextlib import contextmanager +from itertools import chain from pathlib import Path from typing import TYPE_CHECKING, Literal @@ -209,6 +211,11 @@ def recursive_replace_linear( _recursive_replace(model, prefix=prefix) +def named_state(module: nn.Module) -> Iterator[tuple[str, torch.Tensor]]: + """`module`'s own state (i.e. named parameters and buffers).""" + return chain(module.named_parameters(), module.named_buffers()) + + def log_replacement(name: str, old_module: nn.Module, new_module: nn.Module): logger.debug("%s: %s -> %s", name, old_module, new_module)