[Transformers backend] Fuse MoE routers that score after the top-k (#57410)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Harry Mellor
2026-10-13 08:41:23 +00:00
committed by GitHub
co-authored by Claude Opus 5
parent 24aa1191f4
commit 6f11c220b5
6 changed files with 212 additions and 59 deletions
@@ -29,6 +29,42 @@ class TopKRouter(nn.Module):
return logits, value, index
class SelectionScoredRouter(TopKRouter):
"""Scores the selected logits only (GraniteMoE/Aria style), in fp32."""
def forward(self, hidden_states):
logits = F.linear(hidden_states, self.weight).float()
value, index = torch.topk(logits, self.top_k, dim=-1)
value = (
torch.sigmoid(value) if self.sigmoid else F.softmax(value, dim=-1)
).type_as(hidden_states)
return logits, value, index
class EpsilonRenormRouter(TopKRouter):
"""Renormalizes via an epsilon-offset denominator (DeepSeek-V3 style)."""
def forward(self, hidden_states):
logits = F.linear(hidden_states, self.weight)
scores = torch.sigmoid(logits)
value, index = torch.topk(scores, self.top_k, dim=-1)
denominator = value.sum(dim=-1, keepdim=True) + 1e-20
value /= denominator
return logits, value, index
class UntraceableRouter(TopKRouter):
"""Scores and selects, then loses the trace before the renormalization."""
def forward(self, hidden_states):
logits = F.linear(hidden_states, self.weight)
scores = F.softmax(logits, dim=-1)
value, index = torch.topk(scores, self.top_k, dim=-1)
if value.sum() > 0: # data-dependent branch: tracing stops here
value = value / value.sum(dim=-1, keepdim=True)
return logits, value, index
class ScaledRouter(TopKRouter):
"""Greedy router scaling its top-k weights (DeepSeek `routed_scaling_factor`)."""
@@ -268,6 +304,19 @@ def test_moe_fuser_detects_router(sigmoid):
assert fuser.shared_name is None and fuser.shared_gate_name is None
@pytest.mark.parametrize("sigmoid", [False, True])
def test_moe_fuser_matches_router_scoring_the_selection(sigmoid):
"""Scoring the selection rather than all logits still allows the fuser to match."""
with torch.device("meta"):
block = MoEBlock(lambda: SelectionScoredRouter(sigmoid=sigmoid))
fuser = MoEBlockFuser.match(block, "experts")
assert isinstance(fuser, MoEBlockFuser)
assert fuser.gate_name == "gate"
assert fuser.scoring_func == ("sigmoid" if sigmoid else "softmax")
assert fuser.renormalize is (not sigmoid)
assert fuser.router_dtype == torch.float32
def test_moe_fuser_matches_scaled_router():
"""Weight scaling after the top-k (DeepSeek style) does not break matching."""
with torch.device("meta"):
@@ -275,6 +324,27 @@ def test_moe_fuser_matches_scaled_router():
assert isinstance(MoEBlockFuser.match(block, "experts"), MoEBlockFuser)
@pytest.mark.parametrize(
"router_cls,expected",
[
(TopKRouter, True), # divides by the sum of the selection
(EpsilonRenormRouter, True), # ... with an epsilon in the denominator
(ScaledRouter, False), # scales the weights instead of renormalizing
(UntraceableRouter, None), # trace stops before it could say
],
)
def test_moe_fuser_reads_renormalize_from_the_gate(router_cls, expected):
"""The router's graph, not the config, decides whether vLLM renormalizes: it
is the code being replaced, and it already bakes in `norm_topk_prob` because
the flag is a Python bool that fx resolves at trace time. Absence of the
division only means "no" on a complete trace."""
with torch.device("meta"):
block = MoEBlock(router_cls)
fuser = MoEBlockFuser.match(block, "experts")
assert isinstance(fuser, MoEBlockFuser)
assert fuser.renormalize is expected
def test_moe_fuser_matches_grouped_router():
"""Group masking between the score and the routing top-k still matches: the
scorer is anchored on the routing (last) top-k, not the group one."""
@@ -17,6 +17,7 @@ from vllm.model_executor.layers.fused_moe import GateLinear
from vllm.model_executor.layers.linear import ReplicatedLinear
from vllm.model_executor.models.transformers.fx_utils import (
find_node,
is_complete,
is_op,
peel,
trace,
@@ -107,6 +108,24 @@ def _reaches(node: fx.Node, key: str) -> set[fx.Node]:
return seen
def _renormalizes(topk: fx.Node) -> bool:
"""Are the top-k weights divided by their own sum?
Every HF router spells renormalization as a division by a sum over the
selection, optionally offset by an epsilon."""
downstream = _reaches(topk, "users")
for node in downstream:
if not (is_op(node, "div") or is_op(node, "truediv")) or len(node.args) < 2:
continue
denominator = node.args[1]
if not isinstance(denominator, fx.Node):
continue
cone = _reaches(denominator, "all_input_nodes")
if any(is_op(n, "sum") and n in downstream for n in cone):
return True
return False
class SharedExpertMLP(nn.Module):
"""Wraps an HF shared expert, applying the output gating it is paired with."""
@@ -147,12 +166,18 @@ class MoEBlockFuser:
shared_name: str | None
shared_gate_name: str | None
router_dtype: torch.dtype | None = None
renormalize: bool | None = None
@staticmethod
def _match_router(gate: nn.Module) -> tuple[str, torch.dtype | None] | None:
"""Matches `topk(score(linear(x)))`, `score` being `softmax`/`sigmoid`.
def _match_router(
gate: nn.Module,
) -> tuple[str, torch.dtype | None, bool | None] | None:
"""Matches `topk(score(linear(x)))` or `score(topk(linear(x)))`, `score` being
`softmax`/`sigmoid`.
Returns the scoring function and the dtype the router computes in."""
Returns the scoring function, the dtype the router computes in, and
whether the top-k weights are renormalized (`None` if the trace was cut
short before it could say)."""
state = {name for name, _ in named_state(gate)}
if "weight" not in state or state - {"weight", "e_score_correction_bias"}:
return None
@@ -164,20 +189,31 @@ class MoEBlockFuser:
if not topks:
return None
topk = topks[-1]
# Exactly one scoring op upstream of the top-k, fed (transitively) by a linear.
scorers = [
n
for n in _reaches(topk, "all_input_nodes")
if is_op(n, "softmax") or is_op(n, "sigmoid")
]
is_scorer = lambda n: is_op(n, "softmax") or is_op(n, "sigmoid")
post_topk = False
# Find all scoring functions that reach the top-k node.
scorers = [n for n in _reaches(topk, "all_input_nodes") if is_scorer(n)]
if not scorers:
# Find all scoring functions that are users of the top-k node.
scorers = [n for n in _reaches(topk, "users") if is_scorer(n)]
post_topk = True
if len(scorers) != 1:
return None
scorer = scorers[0]
logits_cone = _reaches(scorer, "all_input_nodes")
if not any(is_op(n, "linear") for n in logits_cone):
return None
scoring_func = "softmax" if is_op(scorer, "softmax") else "sigmoid"
return scoring_func, _forced_dtype(logits_cone)
is_softmax = is_op(scorer, "softmax")
# vLLM always scores every expert, so matching a router that softmaxes
# the selection alone takes renormalizing. Otherwise look for the
# division, reading its absence as "no" only from a complete trace.
renormalize: bool | None
if (post_topk and is_softmax) or _renormalizes(topk):
renormalize = True
else:
renormalize = False if is_complete(graph) else None
scoring_func = "softmax" if is_softmax else "sigmoid"
return scoring_func, _forced_dtype(logits_cone), renormalize
@staticmethod
def _match_shared_experts(
@@ -221,14 +257,14 @@ class MoEBlockFuser:
if _returns_tuple(type(moe_block)):
return None
# Router: the child that scores + top-k selects.
gate_name = scoring_func = router_dtype = None
gate_name = scoring_func = router_dtype = renormalize = None
for name, child in moe_block.named_children():
if (
name != experts_name
and (router := cls._match_router(child)) is not None
):
gate_name = name
scoring_func, router_dtype = router
scoring_func, router_dtype, renormalize = router
break
if gate_name is None or scoring_func is None:
return None
@@ -256,7 +292,14 @@ class MoEBlockFuser:
for name, child in moe_block.named_children():
if name not in accounted and next(named_state(child), None) is not None:
return None
return cls(gate_name, scoring_func, shared_name, shared_gate_name, router_dtype)
return cls(
gate_name,
scoring_func,
shared_name,
shared_gate_name,
router_dtype,
renormalize,
)
def gate(
self, moe_block: nn.Module, prefix: str, out_dtype: torch.dtype | None = None
@@ -35,6 +35,21 @@ _MODULE_CALL = nn.Module.__call__
"""The unpatched `nn.Module.__call__`. During tracing fx patches it to record
`call_module` nodes; meta execution must call modules for real."""
_MODULE_GETATTR = nn.Module.__getattr__
"""The unpatched `nn.Module.__getattr__`. During tracing fx patches it to record
`get_attr` nodes; reading one while inferring a `get_attr`'s own value would
recurse until the stack runs out, truncating the graph."""
@contextlib.contextmanager
def _unpatched_modules():
"""Restores the module hooks fx patches, so meta execution records nothing."""
with (
mock.patch.object(nn.Module, "__call__", _MODULE_CALL),
mock.patch.object(nn.Module, "__getattr__", _MODULE_GETATTR),
):
yield
def is_leaf_call(node: object) -> TypeIs[fx.Node]:
"""Is node a call recorded by `_as_leaf_call` (e.g. an attention interface)."""
@@ -132,7 +147,8 @@ class _AllLeafTracer(fx.Tracer):
)
return _UNKNOWN
if kind == "get_attr":
value = operator.attrgetter(str(target))(self.root)
with _unpatched_modules():
value = operator.attrgetter(str(target))(self.root)
if isinstance(value, torch.Tensor):
value = torch.empty_like(value, device="meta")
return value
@@ -159,8 +175,6 @@ class _AllLeafTracer(fx.Tracer):
if kind == "call_module":
# Run the child's forward with all its state on "meta", without
# mutating it (at match time params may be meta but buffers real).
# fx patches `nn.Module.__call__` while tracing; restore the real
# one so this execution is not itself recorded.
child = self.root.get_submodule(str(target))
state = {
name: torch.empty_like(tensor, device="meta")
@@ -168,7 +182,7 @@ class _AllLeafTracer(fx.Tracer):
child.named_parameters(), child.named_buffers()
)
}
with mock.patch.object(nn.Module, "__call__", _MODULE_CALL):
with _unpatched_modules():
return torch.func.functional_call(child, state, meta_args, meta_kwargs)
return _UNKNOWN
@@ -686,6 +700,14 @@ def replace_expr(module: ast.AST, old: ast.expr, new: ast.expr) -> None:
_Replacer().visit(module)
def is_complete(graph: fx.Graph) -> bool:
"""Did the trace run to completion? A partial graph has no `output` node.
Only a complete graph can be read for the *absence* of an op; in a partial
one the tail is exactly what is missing."""
return any(node.op == "output" for node in graph.nodes)
def find_node(graph: fx.Graph, predicate: Callable[[fx.Node], bool]) -> fx.Node | None:
"""The first node in `graph` matching `predicate`, or `None`."""
return next((n for n in graph.nodes if predicate(n)), None)
+18 -13
View File
@@ -39,6 +39,7 @@ from vllm.model_executor.models.transformers.fuser import get_fuser
from vllm.model_executor.models.transformers.fusers.glu import GLUFuser
from vllm.model_executor.models.transformers.fusers.moe import MoEBlockFuser
from vllm.model_executor.models.utils import extract_layer_index, maybe_prefix
from vllm.transformers_utils.config import CONFIG_FIELD_ALIASES
from vllm.utils.torch_utils import STR_DTYPE_TO_TORCH_DTYPE, direct_register_custom_op
from .base import Base
@@ -180,23 +181,18 @@ class MoEMixin(MixtureOfExperts, Base):
text_config = self.text_config
# Positional arguments
num_experts = self.model_config.get_num_experts()
top_k = getattr_iter(text_config, ["num_experts_per_tok", "top_k"], None)
num_experts_names = CONFIG_FIELD_ALIASES["num_experts"]
num_experts = getattr_iter(text_config, num_experts_names, None)
top_k_names = CONFIG_FIELD_ALIASES["num_experts_per_token"]
top_k = getattr_iter(text_config, top_k_names, None)
assert top_k is not None
hidden_size = text_config.hidden_size
intermediate_size = getattr_iter(
text_config, ["moe_intermediate_size", "intermediate_size"], None
)
intermediate_size_names = CONFIG_FIELD_ALIASES["moe_intermediate_size"]
intermediate_size = getattr_iter(text_config, intermediate_size_names, None)
assert intermediate_size is not None
num_shared_experts = getattr_iter(
text_config,
[
"n_shared_experts", # DeepSeek, Docs, GLM
"moe_num_shared_experts", # Aria, Ernie
],
0,
)
num_shared_experts_names = CONFIG_FIELD_ALIASES["num_shared_experts"]
num_shared_experts = getattr_iter(text_config, num_shared_experts_names, 0)
# Common kwargs
norm_topk_prob = getattr(text_config, "norm_topk_prob", None)
@@ -275,6 +271,9 @@ class MoEMixin(MixtureOfExperts, Base):
if "bias" in experts_param_name:
has_bias = True
break
# Whether the expert weights are stored in transposed format.
# Comes from Transformers `use_experts_implementation` decorator.
is_transposed = getattr(experts, "is_transposed", False)
# If the config does not specify num_shared_experts, but
# the model has shared experts, we assume there is one.
if self.num_shared_experts == 0:
@@ -308,6 +307,7 @@ class MoEMixin(MixtureOfExperts, Base):
enable_eplb=enable_eplb,
num_redundant_experts=num_redundant_experts,
has_bias=has_bias,
is_fused_checkpoint_transposed=is_transposed,
routed_experts_cls=TransformersRoutedExperts,
)
fuser = MoEBlockFuser.match(moe_block, experts_name)
@@ -346,6 +346,11 @@ class MoEMixin(MixtureOfExperts, Base):
gate = fuser.gate(moe_block, prefix, router_dtype)
kwargs |= dict(
scoring_func=fuser.scoring_func,
renormalize=(
layer_renormalize
if fuser.renormalize is None
else fuser.renormalize
),
is_sequence_parallel=(
self.parallel_config.use_sequence_parallel_moe
),
+36
View File
@@ -52,6 +52,42 @@ MISTRAL_CONFIG_NAME = "params.json"
logger = init_logger(__name__)
CONFIG_FIELD_ALIASES: dict[str, tuple[str, ...]] = {
"total_num_kv_heads": (
"n_head_kv", # Falcon
"num_kv_heads",
"num_key_value_heads", # LLaMA-2
"multi_query_group_num", # ChatGLM
"num_attention_groups", # Step3p5
),
"num_experts": (
"num_experts", # Jamba
"moe_num_experts", # Dbrx
"n_routed_experts", # DeepSeek
"num_local_experts", # Mixtral
),
"num_experts_per_token": (
"num_experts_per_tok",
"num_experts_per_token",
"top_k_experts",
"moe_topk", # Hunyuan
"moe_top_k",
"top_k",
),
"moe_intermediate_size": (
"moe_intermediate_size",
"intermediate_size",
),
"num_shared_experts": (
"n_shared_experts", # DeepSeek, Docs, GLM
"moe_num_shared_experts", # Aria, Ernie
),
}
"""HF config fields that models spell differently, keyed by the name vLLM uses.
`ModelArchConfigConvertorBase` reads a scalar from these; a caller that needs the
raw value (a per-layer `list`, say) passes the aliases to `getattr_iter` itself."""
_ST_POOLING_MODULE_TYPES = {
"sentence_transformers.models.Pooling",
"sentence_transformers.sentence_transformer.modules.pooling.Pooling",
@@ -14,6 +14,7 @@ from vllm.config.model_arch import (
from vllm.config.utils import getattr_iter
from vllm.logger import init_logger
from vllm.transformers_utils.config import (
CONFIG_FIELD_ALIASES,
ConfigFormat,
get_safetensors_params_metadata,
)
@@ -148,23 +149,11 @@ class ModelArchConfigConvertorBase:
return qk_rope_head_dim
def get_total_num_kv_heads(self) -> int:
attributes = [
# For Falcon:
"n_head_kv",
"num_kv_heads",
# For LLaMA-2:
"num_key_value_heads",
# For ChatGLM:
"multi_query_group_num",
# For Step3p5:
"num_attention_groups",
]
names = CONFIG_FIELD_ALIASES["total_num_kv_heads"]
# For non-grouped-query attention models, the number of KV heads is
# equal to the number of attention heads.
default_factory = self.get_total_num_attention_heads
return getattr_iter(
self.hf_text_config, attributes, default_factory=default_factory
)
return getattr_iter(self.hf_text_config, names, default_factory=default_factory)
def get_num_experts_from_block_configs(self) -> int:
"""Check block_configs for heterogeneous models (e.g., NemotronH).
@@ -188,13 +177,7 @@ class ModelArchConfigConvertorBase:
def get_num_experts(self) -> int:
"""Returns the number of experts in the model."""
num_expert_names = [
"num_experts", # Jamba
"moe_num_experts", # Dbrx
"n_routed_experts", # DeepSeek
"num_local_experts", # Mixtral
]
num_expert_names = CONFIG_FIELD_ALIASES["num_experts"]
num_experts = getattr_iter(self.hf_text_config, num_expert_names, 0)
if isinstance(num_experts, list):
# Ernie VL's remote code uses list[int]...
@@ -206,13 +189,7 @@ class ModelArchConfigConvertorBase:
return num_experts
def get_num_experts_per_token(self) -> int:
names = [
"num_experts_per_tok",
"num_experts_per_token",
"top_k_experts",
"moe_topk",
"moe_top_k",
]
names = CONFIG_FIELD_ALIASES["num_experts_per_token"]
num_experts_per_token = getattr_iter(self.hf_text_config, names, 0)
if isinstance(num_experts_per_token, list):
return max(num_experts_per_token, default=0)