From 6f11c220b5b0f3b45b2a14c49ebbca2df12d1c80 Mon Sep 17 00:00:00 2001 From: Harry Mellor <19981378+hmellor@users.noreply.github.com> Date: Tue, 13 Oct 2026 09:41:23 +0100 Subject: [PATCH] [Transformers backend] Fuse MoE routers that score after the top-k (#57410) Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: Claude Opus 5 (1M context) --- tests/models/transformers/fusers/test_moe.py | 70 ++++++++++++++++++ .../models/transformers/fusers/moe.py | 71 +++++++++++++++---- .../models/transformers/fx_utils.py | 30 ++++++-- .../model_executor/models/transformers/moe.py | 31 ++++---- vllm/transformers_utils/config.py | 36 ++++++++++ .../model_arch_config_convertor.py | 33 ++------- 6 files changed, 212 insertions(+), 59 deletions(-) diff --git a/tests/models/transformers/fusers/test_moe.py b/tests/models/transformers/fusers/test_moe.py index 7de4589eef..f0e70c896b 100644 --- a/tests/models/transformers/fusers/test_moe.py +++ b/tests/models/transformers/fusers/test_moe.py @@ -29,6 +29,42 @@ class TopKRouter(nn.Module): return logits, value, index +class SelectionScoredRouter(TopKRouter): + """Scores the selected logits only (GraniteMoE/Aria style), in fp32.""" + + def forward(self, hidden_states): + logits = F.linear(hidden_states, self.weight).float() + value, index = torch.topk(logits, self.top_k, dim=-1) + value = ( + torch.sigmoid(value) if self.sigmoid else F.softmax(value, dim=-1) + ).type_as(hidden_states) + return logits, value, index + + +class EpsilonRenormRouter(TopKRouter): + """Renormalizes via an epsilon-offset denominator (DeepSeek-V3 style).""" + + def forward(self, hidden_states): + logits = F.linear(hidden_states, self.weight) + scores = torch.sigmoid(logits) + value, index = torch.topk(scores, self.top_k, dim=-1) + denominator = value.sum(dim=-1, keepdim=True) + 1e-20 + value /= denominator + return logits, value, index + + +class UntraceableRouter(TopKRouter): + """Scores and selects, then loses the trace before the renormalization.""" + + def forward(self, hidden_states): + logits = F.linear(hidden_states, self.weight) + scores = F.softmax(logits, dim=-1) + value, index = torch.topk(scores, self.top_k, dim=-1) + if value.sum() > 0: # data-dependent branch: tracing stops here + value = value / value.sum(dim=-1, keepdim=True) + return logits, value, index + + class ScaledRouter(TopKRouter): """Greedy router scaling its top-k weights (DeepSeek `routed_scaling_factor`).""" @@ -268,6 +304,19 @@ def test_moe_fuser_detects_router(sigmoid): assert fuser.shared_name is None and fuser.shared_gate_name is None +@pytest.mark.parametrize("sigmoid", [False, True]) +def test_moe_fuser_matches_router_scoring_the_selection(sigmoid): + """Scoring the selection rather than all logits still allows the fuser to match.""" + with torch.device("meta"): + block = MoEBlock(lambda: SelectionScoredRouter(sigmoid=sigmoid)) + fuser = MoEBlockFuser.match(block, "experts") + assert isinstance(fuser, MoEBlockFuser) + assert fuser.gate_name == "gate" + assert fuser.scoring_func == ("sigmoid" if sigmoid else "softmax") + assert fuser.renormalize is (not sigmoid) + assert fuser.router_dtype == torch.float32 + + def test_moe_fuser_matches_scaled_router(): """Weight scaling after the top-k (DeepSeek style) does not break matching.""" with torch.device("meta"): @@ -275,6 +324,27 @@ def test_moe_fuser_matches_scaled_router(): assert isinstance(MoEBlockFuser.match(block, "experts"), MoEBlockFuser) +@pytest.mark.parametrize( + "router_cls,expected", + [ + (TopKRouter, True), # divides by the sum of the selection + (EpsilonRenormRouter, True), # ... with an epsilon in the denominator + (ScaledRouter, False), # scales the weights instead of renormalizing + (UntraceableRouter, None), # trace stops before it could say + ], +) +def test_moe_fuser_reads_renormalize_from_the_gate(router_cls, expected): + """The router's graph, not the config, decides whether vLLM renormalizes: it + is the code being replaced, and it already bakes in `norm_topk_prob` because + the flag is a Python bool that fx resolves at trace time. Absence of the + division only means "no" on a complete trace.""" + with torch.device("meta"): + block = MoEBlock(router_cls) + fuser = MoEBlockFuser.match(block, "experts") + assert isinstance(fuser, MoEBlockFuser) + assert fuser.renormalize is expected + + def test_moe_fuser_matches_grouped_router(): """Group masking between the score and the routing top-k still matches: the scorer is anchored on the routing (last) top-k, not the group one.""" diff --git a/vllm/model_executor/models/transformers/fusers/moe.py b/vllm/model_executor/models/transformers/fusers/moe.py index af3a101a0c..06fdac109c 100644 --- a/vllm/model_executor/models/transformers/fusers/moe.py +++ b/vllm/model_executor/models/transformers/fusers/moe.py @@ -17,6 +17,7 @@ from vllm.model_executor.layers.fused_moe import GateLinear from vllm.model_executor.layers.linear import ReplicatedLinear from vllm.model_executor.models.transformers.fx_utils import ( find_node, + is_complete, is_op, peel, trace, @@ -107,6 +108,24 @@ def _reaches(node: fx.Node, key: str) -> set[fx.Node]: return seen +def _renormalizes(topk: fx.Node) -> bool: + """Are the top-k weights divided by their own sum? + + Every HF router spells renormalization as a division by a sum over the + selection, optionally offset by an epsilon.""" + downstream = _reaches(topk, "users") + for node in downstream: + if not (is_op(node, "div") or is_op(node, "truediv")) or len(node.args) < 2: + continue + denominator = node.args[1] + if not isinstance(denominator, fx.Node): + continue + cone = _reaches(denominator, "all_input_nodes") + if any(is_op(n, "sum") and n in downstream for n in cone): + return True + return False + + class SharedExpertMLP(nn.Module): """Wraps an HF shared expert, applying the output gating it is paired with.""" @@ -147,12 +166,18 @@ class MoEBlockFuser: shared_name: str | None shared_gate_name: str | None router_dtype: torch.dtype | None = None + renormalize: bool | None = None @staticmethod - def _match_router(gate: nn.Module) -> tuple[str, torch.dtype | None] | None: - """Matches `topk(score(linear(x)))`, `score` being `softmax`/`sigmoid`. + def _match_router( + gate: nn.Module, + ) -> tuple[str, torch.dtype | None, bool | None] | None: + """Matches `topk(score(linear(x)))` or `score(topk(linear(x)))`, `score` being + `softmax`/`sigmoid`. - Returns the scoring function and the dtype the router computes in.""" + Returns the scoring function, the dtype the router computes in, and + whether the top-k weights are renormalized (`None` if the trace was cut + short before it could say).""" state = {name for name, _ in named_state(gate)} if "weight" not in state or state - {"weight", "e_score_correction_bias"}: return None @@ -164,20 +189,31 @@ class MoEBlockFuser: if not topks: return None topk = topks[-1] - # Exactly one scoring op upstream of the top-k, fed (transitively) by a linear. - scorers = [ - n - for n in _reaches(topk, "all_input_nodes") - if is_op(n, "softmax") or is_op(n, "sigmoid") - ] + is_scorer = lambda n: is_op(n, "softmax") or is_op(n, "sigmoid") + post_topk = False + # Find all scoring functions that reach the top-k node. + scorers = [n for n in _reaches(topk, "all_input_nodes") if is_scorer(n)] + if not scorers: + # Find all scoring functions that are users of the top-k node. + scorers = [n for n in _reaches(topk, "users") if is_scorer(n)] + post_topk = True if len(scorers) != 1: return None scorer = scorers[0] logits_cone = _reaches(scorer, "all_input_nodes") if not any(is_op(n, "linear") for n in logits_cone): return None - scoring_func = "softmax" if is_op(scorer, "softmax") else "sigmoid" - return scoring_func, _forced_dtype(logits_cone) + is_softmax = is_op(scorer, "softmax") + # vLLM always scores every expert, so matching a router that softmaxes + # the selection alone takes renormalizing. Otherwise look for the + # division, reading its absence as "no" only from a complete trace. + renormalize: bool | None + if (post_topk and is_softmax) or _renormalizes(topk): + renormalize = True + else: + renormalize = False if is_complete(graph) else None + scoring_func = "softmax" if is_softmax else "sigmoid" + return scoring_func, _forced_dtype(logits_cone), renormalize @staticmethod def _match_shared_experts( @@ -221,14 +257,14 @@ class MoEBlockFuser: if _returns_tuple(type(moe_block)): return None # Router: the child that scores + top-k selects. - gate_name = scoring_func = router_dtype = None + gate_name = scoring_func = router_dtype = renormalize = None for name, child in moe_block.named_children(): if ( name != experts_name and (router := cls._match_router(child)) is not None ): gate_name = name - scoring_func, router_dtype = router + scoring_func, router_dtype, renormalize = router break if gate_name is None or scoring_func is None: return None @@ -256,7 +292,14 @@ class MoEBlockFuser: for name, child in moe_block.named_children(): if name not in accounted and next(named_state(child), None) is not None: return None - return cls(gate_name, scoring_func, shared_name, shared_gate_name, router_dtype) + return cls( + gate_name, + scoring_func, + shared_name, + shared_gate_name, + router_dtype, + renormalize, + ) def gate( self, moe_block: nn.Module, prefix: str, out_dtype: torch.dtype | None = None diff --git a/vllm/model_executor/models/transformers/fx_utils.py b/vllm/model_executor/models/transformers/fx_utils.py index cb630866a3..818e1ca7bb 100644 --- a/vllm/model_executor/models/transformers/fx_utils.py +++ b/vllm/model_executor/models/transformers/fx_utils.py @@ -35,6 +35,21 @@ _MODULE_CALL = nn.Module.__call__ """The unpatched `nn.Module.__call__`. During tracing fx patches it to record `call_module` nodes; meta execution must call modules for real.""" +_MODULE_GETATTR = nn.Module.__getattr__ +"""The unpatched `nn.Module.__getattr__`. During tracing fx patches it to record +`get_attr` nodes; reading one while inferring a `get_attr`'s own value would +recurse until the stack runs out, truncating the graph.""" + + +@contextlib.contextmanager +def _unpatched_modules(): + """Restores the module hooks fx patches, so meta execution records nothing.""" + with ( + mock.patch.object(nn.Module, "__call__", _MODULE_CALL), + mock.patch.object(nn.Module, "__getattr__", _MODULE_GETATTR), + ): + yield + def is_leaf_call(node: object) -> TypeIs[fx.Node]: """Is node a call recorded by `_as_leaf_call` (e.g. an attention interface).""" @@ -132,7 +147,8 @@ class _AllLeafTracer(fx.Tracer): ) return _UNKNOWN if kind == "get_attr": - value = operator.attrgetter(str(target))(self.root) + with _unpatched_modules(): + value = operator.attrgetter(str(target))(self.root) if isinstance(value, torch.Tensor): value = torch.empty_like(value, device="meta") return value @@ -159,8 +175,6 @@ class _AllLeafTracer(fx.Tracer): if kind == "call_module": # Run the child's forward with all its state on "meta", without # mutating it (at match time params may be meta but buffers real). - # fx patches `nn.Module.__call__` while tracing; restore the real - # one so this execution is not itself recorded. child = self.root.get_submodule(str(target)) state = { name: torch.empty_like(tensor, device="meta") @@ -168,7 +182,7 @@ class _AllLeafTracer(fx.Tracer): child.named_parameters(), child.named_buffers() ) } - with mock.patch.object(nn.Module, "__call__", _MODULE_CALL): + with _unpatched_modules(): return torch.func.functional_call(child, state, meta_args, meta_kwargs) return _UNKNOWN @@ -686,6 +700,14 @@ def replace_expr(module: ast.AST, old: ast.expr, new: ast.expr) -> None: _Replacer().visit(module) +def is_complete(graph: fx.Graph) -> bool: + """Did the trace run to completion? A partial graph has no `output` node. + + Only a complete graph can be read for the *absence* of an op; in a partial + one the tail is exactly what is missing.""" + return any(node.op == "output" for node in graph.nodes) + + def find_node(graph: fx.Graph, predicate: Callable[[fx.Node], bool]) -> fx.Node | None: """The first node in `graph` matching `predicate`, or `None`.""" return next((n for n in graph.nodes if predicate(n)), None) diff --git a/vllm/model_executor/models/transformers/moe.py b/vllm/model_executor/models/transformers/moe.py index 71f5bd066c..953459172d 100644 --- a/vllm/model_executor/models/transformers/moe.py +++ b/vllm/model_executor/models/transformers/moe.py @@ -39,6 +39,7 @@ from vllm.model_executor.models.transformers.fuser import get_fuser from vllm.model_executor.models.transformers.fusers.glu import GLUFuser from vllm.model_executor.models.transformers.fusers.moe import MoEBlockFuser from vllm.model_executor.models.utils import extract_layer_index, maybe_prefix +from vllm.transformers_utils.config import CONFIG_FIELD_ALIASES from vllm.utils.torch_utils import STR_DTYPE_TO_TORCH_DTYPE, direct_register_custom_op from .base import Base @@ -180,23 +181,18 @@ class MoEMixin(MixtureOfExperts, Base): text_config = self.text_config # Positional arguments - num_experts = self.model_config.get_num_experts() - top_k = getattr_iter(text_config, ["num_experts_per_tok", "top_k"], None) + num_experts_names = CONFIG_FIELD_ALIASES["num_experts"] + num_experts = getattr_iter(text_config, num_experts_names, None) + top_k_names = CONFIG_FIELD_ALIASES["num_experts_per_token"] + top_k = getattr_iter(text_config, top_k_names, None) assert top_k is not None hidden_size = text_config.hidden_size - intermediate_size = getattr_iter( - text_config, ["moe_intermediate_size", "intermediate_size"], None - ) + intermediate_size_names = CONFIG_FIELD_ALIASES["moe_intermediate_size"] + intermediate_size = getattr_iter(text_config, intermediate_size_names, None) assert intermediate_size is not None - num_shared_experts = getattr_iter( - text_config, - [ - "n_shared_experts", # DeepSeek, Docs, GLM - "moe_num_shared_experts", # Aria, Ernie - ], - 0, - ) + num_shared_experts_names = CONFIG_FIELD_ALIASES["num_shared_experts"] + num_shared_experts = getattr_iter(text_config, num_shared_experts_names, 0) # Common kwargs norm_topk_prob = getattr(text_config, "norm_topk_prob", None) @@ -275,6 +271,9 @@ class MoEMixin(MixtureOfExperts, Base): if "bias" in experts_param_name: has_bias = True break + # Whether the expert weights are stored in transposed format. + # Comes from Transformers `use_experts_implementation` decorator. + is_transposed = getattr(experts, "is_transposed", False) # If the config does not specify num_shared_experts, but # the model has shared experts, we assume there is one. if self.num_shared_experts == 0: @@ -308,6 +307,7 @@ class MoEMixin(MixtureOfExperts, Base): enable_eplb=enable_eplb, num_redundant_experts=num_redundant_experts, has_bias=has_bias, + is_fused_checkpoint_transposed=is_transposed, routed_experts_cls=TransformersRoutedExperts, ) fuser = MoEBlockFuser.match(moe_block, experts_name) @@ -346,6 +346,11 @@ class MoEMixin(MixtureOfExperts, Base): gate = fuser.gate(moe_block, prefix, router_dtype) kwargs |= dict( scoring_func=fuser.scoring_func, + renormalize=( + layer_renormalize + if fuser.renormalize is None + else fuser.renormalize + ), is_sequence_parallel=( self.parallel_config.use_sequence_parallel_moe ), diff --git a/vllm/transformers_utils/config.py b/vllm/transformers_utils/config.py index e046c301d2..1d93a04e66 100644 --- a/vllm/transformers_utils/config.py +++ b/vllm/transformers_utils/config.py @@ -52,6 +52,42 @@ MISTRAL_CONFIG_NAME = "params.json" logger = init_logger(__name__) +CONFIG_FIELD_ALIASES: dict[str, tuple[str, ...]] = { + "total_num_kv_heads": ( + "n_head_kv", # Falcon + "num_kv_heads", + "num_key_value_heads", # LLaMA-2 + "multi_query_group_num", # ChatGLM + "num_attention_groups", # Step3p5 + ), + "num_experts": ( + "num_experts", # Jamba + "moe_num_experts", # Dbrx + "n_routed_experts", # DeepSeek + "num_local_experts", # Mixtral + ), + "num_experts_per_token": ( + "num_experts_per_tok", + "num_experts_per_token", + "top_k_experts", + "moe_topk", # Hunyuan + "moe_top_k", + "top_k", + ), + "moe_intermediate_size": ( + "moe_intermediate_size", + "intermediate_size", + ), + "num_shared_experts": ( + "n_shared_experts", # DeepSeek, Docs, GLM + "moe_num_shared_experts", # Aria, Ernie + ), +} +"""HF config fields that models spell differently, keyed by the name vLLM uses. + +`ModelArchConfigConvertorBase` reads a scalar from these; a caller that needs the +raw value (a per-layer `list`, say) passes the aliases to `getattr_iter` itself.""" + _ST_POOLING_MODULE_TYPES = { "sentence_transformers.models.Pooling", "sentence_transformers.sentence_transformer.modules.pooling.Pooling", diff --git a/vllm/transformers_utils/model_arch_config_convertor.py b/vllm/transformers_utils/model_arch_config_convertor.py index 06a9376971..6290f6c664 100644 --- a/vllm/transformers_utils/model_arch_config_convertor.py +++ b/vllm/transformers_utils/model_arch_config_convertor.py @@ -14,6 +14,7 @@ from vllm.config.model_arch import ( from vllm.config.utils import getattr_iter from vllm.logger import init_logger from vllm.transformers_utils.config import ( + CONFIG_FIELD_ALIASES, ConfigFormat, get_safetensors_params_metadata, ) @@ -148,23 +149,11 @@ class ModelArchConfigConvertorBase: return qk_rope_head_dim def get_total_num_kv_heads(self) -> int: - attributes = [ - # For Falcon: - "n_head_kv", - "num_kv_heads", - # For LLaMA-2: - "num_key_value_heads", - # For ChatGLM: - "multi_query_group_num", - # For Step3p5: - "num_attention_groups", - ] + names = CONFIG_FIELD_ALIASES["total_num_kv_heads"] # For non-grouped-query attention models, the number of KV heads is # equal to the number of attention heads. default_factory = self.get_total_num_attention_heads - return getattr_iter( - self.hf_text_config, attributes, default_factory=default_factory - ) + return getattr_iter(self.hf_text_config, names, default_factory=default_factory) def get_num_experts_from_block_configs(self) -> int: """Check block_configs for heterogeneous models (e.g., NemotronH). @@ -188,13 +177,7 @@ class ModelArchConfigConvertorBase: def get_num_experts(self) -> int: """Returns the number of experts in the model.""" - num_expert_names = [ - "num_experts", # Jamba - "moe_num_experts", # Dbrx - "n_routed_experts", # DeepSeek - "num_local_experts", # Mixtral - ] - + num_expert_names = CONFIG_FIELD_ALIASES["num_experts"] num_experts = getattr_iter(self.hf_text_config, num_expert_names, 0) if isinstance(num_experts, list): # Ernie VL's remote code uses list[int]... @@ -206,13 +189,7 @@ class ModelArchConfigConvertorBase: return num_experts def get_num_experts_per_token(self) -> int: - names = [ - "num_experts_per_tok", - "num_experts_per_token", - "top_k_experts", - "moe_topk", - "moe_top_k", - ] + names = CONFIG_FIELD_ALIASES["num_experts_per_token"] num_experts_per_token = getattr_iter(self.hf_text_config, names, 0) if isinstance(num_experts_per_token, list): return max(num_experts_per_token, default=0)