[Transformers backend] Fuse MoE routers that score after the top-k (#57410)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Harry Mellor
2026-10-13 08:41:23 +00:00
committed by GitHub
co-authored by Claude Opus 5
parent 24aa1191f4
commit 6f11c220b5
6 changed files with 212 additions and 59 deletions
@@ -17,6 +17,7 @@ from vllm.model_executor.layers.fused_moe import GateLinear
from vllm.model_executor.layers.linear import ReplicatedLinear
from vllm.model_executor.models.transformers.fx_utils import (
find_node,
is_complete,
is_op,
peel,
trace,
@@ -107,6 +108,24 @@ def _reaches(node: fx.Node, key: str) -> set[fx.Node]:
return seen
def _renormalizes(topk: fx.Node) -> bool:
"""Are the top-k weights divided by their own sum?
Every HF router spells renormalization as a division by a sum over the
selection, optionally offset by an epsilon."""
downstream = _reaches(topk, "users")
for node in downstream:
if not (is_op(node, "div") or is_op(node, "truediv")) or len(node.args) < 2:
continue
denominator = node.args[1]
if not isinstance(denominator, fx.Node):
continue
cone = _reaches(denominator, "all_input_nodes")
if any(is_op(n, "sum") and n in downstream for n in cone):
return True
return False
class SharedExpertMLP(nn.Module):
"""Wraps an HF shared expert, applying the output gating it is paired with."""
@@ -147,12 +166,18 @@ class MoEBlockFuser:
shared_name: str | None
shared_gate_name: str | None
router_dtype: torch.dtype | None = None
renormalize: bool | None = None
@staticmethod
def _match_router(gate: nn.Module) -> tuple[str, torch.dtype | None] | None:
"""Matches `topk(score(linear(x)))`, `score` being `softmax`/`sigmoid`.
def _match_router(
gate: nn.Module,
) -> tuple[str, torch.dtype | None, bool | None] | None:
"""Matches `topk(score(linear(x)))` or `score(topk(linear(x)))`, `score` being
`softmax`/`sigmoid`.
Returns the scoring function and the dtype the router computes in."""
Returns the scoring function, the dtype the router computes in, and
whether the top-k weights are renormalized (`None` if the trace was cut
short before it could say)."""
state = {name for name, _ in named_state(gate)}
if "weight" not in state or state - {"weight", "e_score_correction_bias"}:
return None
@@ -164,20 +189,31 @@ class MoEBlockFuser:
if not topks:
return None
topk = topks[-1]
# Exactly one scoring op upstream of the top-k, fed (transitively) by a linear.
scorers = [
n
for n in _reaches(topk, "all_input_nodes")
if is_op(n, "softmax") or is_op(n, "sigmoid")
]
is_scorer = lambda n: is_op(n, "softmax") or is_op(n, "sigmoid")
post_topk = False
# Find all scoring functions that reach the top-k node.
scorers = [n for n in _reaches(topk, "all_input_nodes") if is_scorer(n)]
if not scorers:
# Find all scoring functions that are users of the top-k node.
scorers = [n for n in _reaches(topk, "users") if is_scorer(n)]
post_topk = True
if len(scorers) != 1:
return None
scorer = scorers[0]
logits_cone = _reaches(scorer, "all_input_nodes")
if not any(is_op(n, "linear") for n in logits_cone):
return None
scoring_func = "softmax" if is_op(scorer, "softmax") else "sigmoid"
return scoring_func, _forced_dtype(logits_cone)
is_softmax = is_op(scorer, "softmax")
# vLLM always scores every expert, so matching a router that softmaxes
# the selection alone takes renormalizing. Otherwise look for the
# division, reading its absence as "no" only from a complete trace.
renormalize: bool | None
if (post_topk and is_softmax) or _renormalizes(topk):
renormalize = True
else:
renormalize = False if is_complete(graph) else None
scoring_func = "softmax" if is_softmax else "sigmoid"
return scoring_func, _forced_dtype(logits_cone), renormalize
@staticmethod
def _match_shared_experts(
@@ -221,14 +257,14 @@ class MoEBlockFuser:
if _returns_tuple(type(moe_block)):
return None
# Router: the child that scores + top-k selects.
gate_name = scoring_func = router_dtype = None
gate_name = scoring_func = router_dtype = renormalize = None
for name, child in moe_block.named_children():
if (
name != experts_name
and (router := cls._match_router(child)) is not None
):
gate_name = name
scoring_func, router_dtype = router
scoring_func, router_dtype, renormalize = router
break
if gate_name is None or scoring_func is None:
return None
@@ -256,7 +292,14 @@ class MoEBlockFuser:
for name, child in moe_block.named_children():
if name not in accounted and next(named_state(child), None) is not None:
return None
return cls(gate_name, scoring_func, shared_name, shared_gate_name, router_dtype)
return cls(
gate_name,
scoring_func,
shared_name,
shared_gate_name,
router_dtype,
renormalize,
)
def gate(
self, moe_block: nn.Module, prefix: str, out_dtype: torch.dtype | None = None
@@ -35,6 +35,21 @@ _MODULE_CALL = nn.Module.__call__
"""The unpatched `nn.Module.__call__`. During tracing fx patches it to record
`call_module` nodes; meta execution must call modules for real."""
_MODULE_GETATTR = nn.Module.__getattr__
"""The unpatched `nn.Module.__getattr__`. During tracing fx patches it to record
`get_attr` nodes; reading one while inferring a `get_attr`'s own value would
recurse until the stack runs out, truncating the graph."""
@contextlib.contextmanager
def _unpatched_modules():
"""Restores the module hooks fx patches, so meta execution records nothing."""
with (
mock.patch.object(nn.Module, "__call__", _MODULE_CALL),
mock.patch.object(nn.Module, "__getattr__", _MODULE_GETATTR),
):
yield
def is_leaf_call(node: object) -> TypeIs[fx.Node]:
"""Is node a call recorded by `_as_leaf_call` (e.g. an attention interface)."""
@@ -132,7 +147,8 @@ class _AllLeafTracer(fx.Tracer):
)
return _UNKNOWN
if kind == "get_attr":
value = operator.attrgetter(str(target))(self.root)
with _unpatched_modules():
value = operator.attrgetter(str(target))(self.root)
if isinstance(value, torch.Tensor):
value = torch.empty_like(value, device="meta")
return value
@@ -159,8 +175,6 @@ class _AllLeafTracer(fx.Tracer):
if kind == "call_module":
# Run the child's forward with all its state on "meta", without
# mutating it (at match time params may be meta but buffers real).
# fx patches `nn.Module.__call__` while tracing; restore the real
# one so this execution is not itself recorded.
child = self.root.get_submodule(str(target))
state = {
name: torch.empty_like(tensor, device="meta")
@@ -168,7 +182,7 @@ class _AllLeafTracer(fx.Tracer):
child.named_parameters(), child.named_buffers()
)
}
with mock.patch.object(nn.Module, "__call__", _MODULE_CALL):
with _unpatched_modules():
return torch.func.functional_call(child, state, meta_args, meta_kwargs)
return _UNKNOWN
@@ -686,6 +700,14 @@ def replace_expr(module: ast.AST, old: ast.expr, new: ast.expr) -> None:
_Replacer().visit(module)
def is_complete(graph: fx.Graph) -> bool:
"""Did the trace run to completion? A partial graph has no `output` node.
Only a complete graph can be read for the *absence* of an op; in a partial
one the tail is exactly what is missing."""
return any(node.op == "output" for node in graph.nodes)
def find_node(graph: fx.Graph, predicate: Callable[[fx.Node], bool]) -> fx.Node | None:
"""The first node in `graph` matching `predicate`, or `None`."""
return next((n for n in graph.nodes if predicate(n)), None)
+18 -13
View File
@@ -39,6 +39,7 @@ from vllm.model_executor.models.transformers.fuser import get_fuser
from vllm.model_executor.models.transformers.fusers.glu import GLUFuser
from vllm.model_executor.models.transformers.fusers.moe import MoEBlockFuser
from vllm.model_executor.models.utils import extract_layer_index, maybe_prefix
from vllm.transformers_utils.config import CONFIG_FIELD_ALIASES
from vllm.utils.torch_utils import STR_DTYPE_TO_TORCH_DTYPE, direct_register_custom_op
from .base import Base
@@ -180,23 +181,18 @@ class MoEMixin(MixtureOfExperts, Base):
text_config = self.text_config
# Positional arguments
num_experts = self.model_config.get_num_experts()
top_k = getattr_iter(text_config, ["num_experts_per_tok", "top_k"], None)
num_experts_names = CONFIG_FIELD_ALIASES["num_experts"]
num_experts = getattr_iter(text_config, num_experts_names, None)
top_k_names = CONFIG_FIELD_ALIASES["num_experts_per_token"]
top_k = getattr_iter(text_config, top_k_names, None)
assert top_k is not None
hidden_size = text_config.hidden_size
intermediate_size = getattr_iter(
text_config, ["moe_intermediate_size", "intermediate_size"], None
)
intermediate_size_names = CONFIG_FIELD_ALIASES["moe_intermediate_size"]
intermediate_size = getattr_iter(text_config, intermediate_size_names, None)
assert intermediate_size is not None
num_shared_experts = getattr_iter(
text_config,
[
"n_shared_experts", # DeepSeek, Docs, GLM
"moe_num_shared_experts", # Aria, Ernie
],
0,
)
num_shared_experts_names = CONFIG_FIELD_ALIASES["num_shared_experts"]
num_shared_experts = getattr_iter(text_config, num_shared_experts_names, 0)
# Common kwargs
norm_topk_prob = getattr(text_config, "norm_topk_prob", None)
@@ -275,6 +271,9 @@ class MoEMixin(MixtureOfExperts, Base):
if "bias" in experts_param_name:
has_bias = True
break
# Whether the expert weights are stored in transposed format.
# Comes from Transformers `use_experts_implementation` decorator.
is_transposed = getattr(experts, "is_transposed", False)
# If the config does not specify num_shared_experts, but
# the model has shared experts, we assume there is one.
if self.num_shared_experts == 0:
@@ -308,6 +307,7 @@ class MoEMixin(MixtureOfExperts, Base):
enable_eplb=enable_eplb,
num_redundant_experts=num_redundant_experts,
has_bias=has_bias,
is_fused_checkpoint_transposed=is_transposed,
routed_experts_cls=TransformersRoutedExperts,
)
fuser = MoEBlockFuser.match(moe_block, experts_name)
@@ -346,6 +346,11 @@ class MoEMixin(MixtureOfExperts, Base):
gate = fuser.gate(moe_block, prefix, router_dtype)
kwargs |= dict(
scoring_func=fuser.scoring_func,
renormalize=(
layer_renormalize
if fuser.renormalize is None
else fuser.renormalize
),
is_sequence_parallel=(
self.parallel_config.use_sequence_parallel_moe
),