[Transformers backend] Fuse MoE routers that score after the top-k (#57410)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5
parent
24aa1191f4
commit
6f11c220b5
@@ -29,6 +29,42 @@ class TopKRouter(nn.Module):
|
||||
return logits, value, index
|
||||
|
||||
|
||||
class SelectionScoredRouter(TopKRouter):
|
||||
"""Scores the selected logits only (GraniteMoE/Aria style), in fp32."""
|
||||
|
||||
def forward(self, hidden_states):
|
||||
logits = F.linear(hidden_states, self.weight).float()
|
||||
value, index = torch.topk(logits, self.top_k, dim=-1)
|
||||
value = (
|
||||
torch.sigmoid(value) if self.sigmoid else F.softmax(value, dim=-1)
|
||||
).type_as(hidden_states)
|
||||
return logits, value, index
|
||||
|
||||
|
||||
class EpsilonRenormRouter(TopKRouter):
|
||||
"""Renormalizes via an epsilon-offset denominator (DeepSeek-V3 style)."""
|
||||
|
||||
def forward(self, hidden_states):
|
||||
logits = F.linear(hidden_states, self.weight)
|
||||
scores = torch.sigmoid(logits)
|
||||
value, index = torch.topk(scores, self.top_k, dim=-1)
|
||||
denominator = value.sum(dim=-1, keepdim=True) + 1e-20
|
||||
value /= denominator
|
||||
return logits, value, index
|
||||
|
||||
|
||||
class UntraceableRouter(TopKRouter):
|
||||
"""Scores and selects, then loses the trace before the renormalization."""
|
||||
|
||||
def forward(self, hidden_states):
|
||||
logits = F.linear(hidden_states, self.weight)
|
||||
scores = F.softmax(logits, dim=-1)
|
||||
value, index = torch.topk(scores, self.top_k, dim=-1)
|
||||
if value.sum() > 0: # data-dependent branch: tracing stops here
|
||||
value = value / value.sum(dim=-1, keepdim=True)
|
||||
return logits, value, index
|
||||
|
||||
|
||||
class ScaledRouter(TopKRouter):
|
||||
"""Greedy router scaling its top-k weights (DeepSeek `routed_scaling_factor`)."""
|
||||
|
||||
@@ -268,6 +304,19 @@ def test_moe_fuser_detects_router(sigmoid):
|
||||
assert fuser.shared_name is None and fuser.shared_gate_name is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize("sigmoid", [False, True])
|
||||
def test_moe_fuser_matches_router_scoring_the_selection(sigmoid):
|
||||
"""Scoring the selection rather than all logits still allows the fuser to match."""
|
||||
with torch.device("meta"):
|
||||
block = MoEBlock(lambda: SelectionScoredRouter(sigmoid=sigmoid))
|
||||
fuser = MoEBlockFuser.match(block, "experts")
|
||||
assert isinstance(fuser, MoEBlockFuser)
|
||||
assert fuser.gate_name == "gate"
|
||||
assert fuser.scoring_func == ("sigmoid" if sigmoid else "softmax")
|
||||
assert fuser.renormalize is (not sigmoid)
|
||||
assert fuser.router_dtype == torch.float32
|
||||
|
||||
|
||||
def test_moe_fuser_matches_scaled_router():
|
||||
"""Weight scaling after the top-k (DeepSeek style) does not break matching."""
|
||||
with torch.device("meta"):
|
||||
@@ -275,6 +324,27 @@ def test_moe_fuser_matches_scaled_router():
|
||||
assert isinstance(MoEBlockFuser.match(block, "experts"), MoEBlockFuser)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"router_cls,expected",
|
||||
[
|
||||
(TopKRouter, True), # divides by the sum of the selection
|
||||
(EpsilonRenormRouter, True), # ... with an epsilon in the denominator
|
||||
(ScaledRouter, False), # scales the weights instead of renormalizing
|
||||
(UntraceableRouter, None), # trace stops before it could say
|
||||
],
|
||||
)
|
||||
def test_moe_fuser_reads_renormalize_from_the_gate(router_cls, expected):
|
||||
"""The router's graph, not the config, decides whether vLLM renormalizes: it
|
||||
is the code being replaced, and it already bakes in `norm_topk_prob` because
|
||||
the flag is a Python bool that fx resolves at trace time. Absence of the
|
||||
division only means "no" on a complete trace."""
|
||||
with torch.device("meta"):
|
||||
block = MoEBlock(router_cls)
|
||||
fuser = MoEBlockFuser.match(block, "experts")
|
||||
assert isinstance(fuser, MoEBlockFuser)
|
||||
assert fuser.renormalize is expected
|
||||
|
||||
|
||||
def test_moe_fuser_matches_grouped_router():
|
||||
"""Group masking between the score and the routing top-k still matches: the
|
||||
scorer is anchored on the routing (last) top-k, not the group one."""
|
||||
|
||||
@@ -17,6 +17,7 @@ from vllm.model_executor.layers.fused_moe import GateLinear
|
||||
from vllm.model_executor.layers.linear import ReplicatedLinear
|
||||
from vllm.model_executor.models.transformers.fx_utils import (
|
||||
find_node,
|
||||
is_complete,
|
||||
is_op,
|
||||
peel,
|
||||
trace,
|
||||
@@ -107,6 +108,24 @@ def _reaches(node: fx.Node, key: str) -> set[fx.Node]:
|
||||
return seen
|
||||
|
||||
|
||||
def _renormalizes(topk: fx.Node) -> bool:
|
||||
"""Are the top-k weights divided by their own sum?
|
||||
|
||||
Every HF router spells renormalization as a division by a sum over the
|
||||
selection, optionally offset by an epsilon."""
|
||||
downstream = _reaches(topk, "users")
|
||||
for node in downstream:
|
||||
if not (is_op(node, "div") or is_op(node, "truediv")) or len(node.args) < 2:
|
||||
continue
|
||||
denominator = node.args[1]
|
||||
if not isinstance(denominator, fx.Node):
|
||||
continue
|
||||
cone = _reaches(denominator, "all_input_nodes")
|
||||
if any(is_op(n, "sum") and n in downstream for n in cone):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
class SharedExpertMLP(nn.Module):
|
||||
"""Wraps an HF shared expert, applying the output gating it is paired with."""
|
||||
|
||||
@@ -147,12 +166,18 @@ class MoEBlockFuser:
|
||||
shared_name: str | None
|
||||
shared_gate_name: str | None
|
||||
router_dtype: torch.dtype | None = None
|
||||
renormalize: bool | None = None
|
||||
|
||||
@staticmethod
|
||||
def _match_router(gate: nn.Module) -> tuple[str, torch.dtype | None] | None:
|
||||
"""Matches `topk(score(linear(x)))`, `score` being `softmax`/`sigmoid`.
|
||||
def _match_router(
|
||||
gate: nn.Module,
|
||||
) -> tuple[str, torch.dtype | None, bool | None] | None:
|
||||
"""Matches `topk(score(linear(x)))` or `score(topk(linear(x)))`, `score` being
|
||||
`softmax`/`sigmoid`.
|
||||
|
||||
Returns the scoring function and the dtype the router computes in."""
|
||||
Returns the scoring function, the dtype the router computes in, and
|
||||
whether the top-k weights are renormalized (`None` if the trace was cut
|
||||
short before it could say)."""
|
||||
state = {name for name, _ in named_state(gate)}
|
||||
if "weight" not in state or state - {"weight", "e_score_correction_bias"}:
|
||||
return None
|
||||
@@ -164,20 +189,31 @@ class MoEBlockFuser:
|
||||
if not topks:
|
||||
return None
|
||||
topk = topks[-1]
|
||||
# Exactly one scoring op upstream of the top-k, fed (transitively) by a linear.
|
||||
scorers = [
|
||||
n
|
||||
for n in _reaches(topk, "all_input_nodes")
|
||||
if is_op(n, "softmax") or is_op(n, "sigmoid")
|
||||
]
|
||||
is_scorer = lambda n: is_op(n, "softmax") or is_op(n, "sigmoid")
|
||||
post_topk = False
|
||||
# Find all scoring functions that reach the top-k node.
|
||||
scorers = [n for n in _reaches(topk, "all_input_nodes") if is_scorer(n)]
|
||||
if not scorers:
|
||||
# Find all scoring functions that are users of the top-k node.
|
||||
scorers = [n for n in _reaches(topk, "users") if is_scorer(n)]
|
||||
post_topk = True
|
||||
if len(scorers) != 1:
|
||||
return None
|
||||
scorer = scorers[0]
|
||||
logits_cone = _reaches(scorer, "all_input_nodes")
|
||||
if not any(is_op(n, "linear") for n in logits_cone):
|
||||
return None
|
||||
scoring_func = "softmax" if is_op(scorer, "softmax") else "sigmoid"
|
||||
return scoring_func, _forced_dtype(logits_cone)
|
||||
is_softmax = is_op(scorer, "softmax")
|
||||
# vLLM always scores every expert, so matching a router that softmaxes
|
||||
# the selection alone takes renormalizing. Otherwise look for the
|
||||
# division, reading its absence as "no" only from a complete trace.
|
||||
renormalize: bool | None
|
||||
if (post_topk and is_softmax) or _renormalizes(topk):
|
||||
renormalize = True
|
||||
else:
|
||||
renormalize = False if is_complete(graph) else None
|
||||
scoring_func = "softmax" if is_softmax else "sigmoid"
|
||||
return scoring_func, _forced_dtype(logits_cone), renormalize
|
||||
|
||||
@staticmethod
|
||||
def _match_shared_experts(
|
||||
@@ -221,14 +257,14 @@ class MoEBlockFuser:
|
||||
if _returns_tuple(type(moe_block)):
|
||||
return None
|
||||
# Router: the child that scores + top-k selects.
|
||||
gate_name = scoring_func = router_dtype = None
|
||||
gate_name = scoring_func = router_dtype = renormalize = None
|
||||
for name, child in moe_block.named_children():
|
||||
if (
|
||||
name != experts_name
|
||||
and (router := cls._match_router(child)) is not None
|
||||
):
|
||||
gate_name = name
|
||||
scoring_func, router_dtype = router
|
||||
scoring_func, router_dtype, renormalize = router
|
||||
break
|
||||
if gate_name is None or scoring_func is None:
|
||||
return None
|
||||
@@ -256,7 +292,14 @@ class MoEBlockFuser:
|
||||
for name, child in moe_block.named_children():
|
||||
if name not in accounted and next(named_state(child), None) is not None:
|
||||
return None
|
||||
return cls(gate_name, scoring_func, shared_name, shared_gate_name, router_dtype)
|
||||
return cls(
|
||||
gate_name,
|
||||
scoring_func,
|
||||
shared_name,
|
||||
shared_gate_name,
|
||||
router_dtype,
|
||||
renormalize,
|
||||
)
|
||||
|
||||
def gate(
|
||||
self, moe_block: nn.Module, prefix: str, out_dtype: torch.dtype | None = None
|
||||
|
||||
@@ -35,6 +35,21 @@ _MODULE_CALL = nn.Module.__call__
|
||||
"""The unpatched `nn.Module.__call__`. During tracing fx patches it to record
|
||||
`call_module` nodes; meta execution must call modules for real."""
|
||||
|
||||
_MODULE_GETATTR = nn.Module.__getattr__
|
||||
"""The unpatched `nn.Module.__getattr__`. During tracing fx patches it to record
|
||||
`get_attr` nodes; reading one while inferring a `get_attr`'s own value would
|
||||
recurse until the stack runs out, truncating the graph."""
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def _unpatched_modules():
|
||||
"""Restores the module hooks fx patches, so meta execution records nothing."""
|
||||
with (
|
||||
mock.patch.object(nn.Module, "__call__", _MODULE_CALL),
|
||||
mock.patch.object(nn.Module, "__getattr__", _MODULE_GETATTR),
|
||||
):
|
||||
yield
|
||||
|
||||
|
||||
def is_leaf_call(node: object) -> TypeIs[fx.Node]:
|
||||
"""Is node a call recorded by `_as_leaf_call` (e.g. an attention interface)."""
|
||||
@@ -132,7 +147,8 @@ class _AllLeafTracer(fx.Tracer):
|
||||
)
|
||||
return _UNKNOWN
|
||||
if kind == "get_attr":
|
||||
value = operator.attrgetter(str(target))(self.root)
|
||||
with _unpatched_modules():
|
||||
value = operator.attrgetter(str(target))(self.root)
|
||||
if isinstance(value, torch.Tensor):
|
||||
value = torch.empty_like(value, device="meta")
|
||||
return value
|
||||
@@ -159,8 +175,6 @@ class _AllLeafTracer(fx.Tracer):
|
||||
if kind == "call_module":
|
||||
# Run the child's forward with all its state on "meta", without
|
||||
# mutating it (at match time params may be meta but buffers real).
|
||||
# fx patches `nn.Module.__call__` while tracing; restore the real
|
||||
# one so this execution is not itself recorded.
|
||||
child = self.root.get_submodule(str(target))
|
||||
state = {
|
||||
name: torch.empty_like(tensor, device="meta")
|
||||
@@ -168,7 +182,7 @@ class _AllLeafTracer(fx.Tracer):
|
||||
child.named_parameters(), child.named_buffers()
|
||||
)
|
||||
}
|
||||
with mock.patch.object(nn.Module, "__call__", _MODULE_CALL):
|
||||
with _unpatched_modules():
|
||||
return torch.func.functional_call(child, state, meta_args, meta_kwargs)
|
||||
return _UNKNOWN
|
||||
|
||||
@@ -686,6 +700,14 @@ def replace_expr(module: ast.AST, old: ast.expr, new: ast.expr) -> None:
|
||||
_Replacer().visit(module)
|
||||
|
||||
|
||||
def is_complete(graph: fx.Graph) -> bool:
|
||||
"""Did the trace run to completion? A partial graph has no `output` node.
|
||||
|
||||
Only a complete graph can be read for the *absence* of an op; in a partial
|
||||
one the tail is exactly what is missing."""
|
||||
return any(node.op == "output" for node in graph.nodes)
|
||||
|
||||
|
||||
def find_node(graph: fx.Graph, predicate: Callable[[fx.Node], bool]) -> fx.Node | None:
|
||||
"""The first node in `graph` matching `predicate`, or `None`."""
|
||||
return next((n for n in graph.nodes if predicate(n)), None)
|
||||
|
||||
@@ -39,6 +39,7 @@ from vllm.model_executor.models.transformers.fuser import get_fuser
|
||||
from vllm.model_executor.models.transformers.fusers.glu import GLUFuser
|
||||
from vllm.model_executor.models.transformers.fusers.moe import MoEBlockFuser
|
||||
from vllm.model_executor.models.utils import extract_layer_index, maybe_prefix
|
||||
from vllm.transformers_utils.config import CONFIG_FIELD_ALIASES
|
||||
from vllm.utils.torch_utils import STR_DTYPE_TO_TORCH_DTYPE, direct_register_custom_op
|
||||
|
||||
from .base import Base
|
||||
@@ -180,23 +181,18 @@ class MoEMixin(MixtureOfExperts, Base):
|
||||
text_config = self.text_config
|
||||
|
||||
# Positional arguments
|
||||
num_experts = self.model_config.get_num_experts()
|
||||
top_k = getattr_iter(text_config, ["num_experts_per_tok", "top_k"], None)
|
||||
num_experts_names = CONFIG_FIELD_ALIASES["num_experts"]
|
||||
num_experts = getattr_iter(text_config, num_experts_names, None)
|
||||
top_k_names = CONFIG_FIELD_ALIASES["num_experts_per_token"]
|
||||
top_k = getattr_iter(text_config, top_k_names, None)
|
||||
assert top_k is not None
|
||||
hidden_size = text_config.hidden_size
|
||||
intermediate_size = getattr_iter(
|
||||
text_config, ["moe_intermediate_size", "intermediate_size"], None
|
||||
)
|
||||
intermediate_size_names = CONFIG_FIELD_ALIASES["moe_intermediate_size"]
|
||||
intermediate_size = getattr_iter(text_config, intermediate_size_names, None)
|
||||
assert intermediate_size is not None
|
||||
|
||||
num_shared_experts = getattr_iter(
|
||||
text_config,
|
||||
[
|
||||
"n_shared_experts", # DeepSeek, Docs, GLM
|
||||
"moe_num_shared_experts", # Aria, Ernie
|
||||
],
|
||||
0,
|
||||
)
|
||||
num_shared_experts_names = CONFIG_FIELD_ALIASES["num_shared_experts"]
|
||||
num_shared_experts = getattr_iter(text_config, num_shared_experts_names, 0)
|
||||
|
||||
# Common kwargs
|
||||
norm_topk_prob = getattr(text_config, "norm_topk_prob", None)
|
||||
@@ -275,6 +271,9 @@ class MoEMixin(MixtureOfExperts, Base):
|
||||
if "bias" in experts_param_name:
|
||||
has_bias = True
|
||||
break
|
||||
# Whether the expert weights are stored in transposed format.
|
||||
# Comes from Transformers `use_experts_implementation` decorator.
|
||||
is_transposed = getattr(experts, "is_transposed", False)
|
||||
# If the config does not specify num_shared_experts, but
|
||||
# the model has shared experts, we assume there is one.
|
||||
if self.num_shared_experts == 0:
|
||||
@@ -308,6 +307,7 @@ class MoEMixin(MixtureOfExperts, Base):
|
||||
enable_eplb=enable_eplb,
|
||||
num_redundant_experts=num_redundant_experts,
|
||||
has_bias=has_bias,
|
||||
is_fused_checkpoint_transposed=is_transposed,
|
||||
routed_experts_cls=TransformersRoutedExperts,
|
||||
)
|
||||
fuser = MoEBlockFuser.match(moe_block, experts_name)
|
||||
@@ -346,6 +346,11 @@ class MoEMixin(MixtureOfExperts, Base):
|
||||
gate = fuser.gate(moe_block, prefix, router_dtype)
|
||||
kwargs |= dict(
|
||||
scoring_func=fuser.scoring_func,
|
||||
renormalize=(
|
||||
layer_renormalize
|
||||
if fuser.renormalize is None
|
||||
else fuser.renormalize
|
||||
),
|
||||
is_sequence_parallel=(
|
||||
self.parallel_config.use_sequence_parallel_moe
|
||||
),
|
||||
|
||||
@@ -52,6 +52,42 @@ MISTRAL_CONFIG_NAME = "params.json"
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
CONFIG_FIELD_ALIASES: dict[str, tuple[str, ...]] = {
|
||||
"total_num_kv_heads": (
|
||||
"n_head_kv", # Falcon
|
||||
"num_kv_heads",
|
||||
"num_key_value_heads", # LLaMA-2
|
||||
"multi_query_group_num", # ChatGLM
|
||||
"num_attention_groups", # Step3p5
|
||||
),
|
||||
"num_experts": (
|
||||
"num_experts", # Jamba
|
||||
"moe_num_experts", # Dbrx
|
||||
"n_routed_experts", # DeepSeek
|
||||
"num_local_experts", # Mixtral
|
||||
),
|
||||
"num_experts_per_token": (
|
||||
"num_experts_per_tok",
|
||||
"num_experts_per_token",
|
||||
"top_k_experts",
|
||||
"moe_topk", # Hunyuan
|
||||
"moe_top_k",
|
||||
"top_k",
|
||||
),
|
||||
"moe_intermediate_size": (
|
||||
"moe_intermediate_size",
|
||||
"intermediate_size",
|
||||
),
|
||||
"num_shared_experts": (
|
||||
"n_shared_experts", # DeepSeek, Docs, GLM
|
||||
"moe_num_shared_experts", # Aria, Ernie
|
||||
),
|
||||
}
|
||||
"""HF config fields that models spell differently, keyed by the name vLLM uses.
|
||||
|
||||
`ModelArchConfigConvertorBase` reads a scalar from these; a caller that needs the
|
||||
raw value (a per-layer `list`, say) passes the aliases to `getattr_iter` itself."""
|
||||
|
||||
_ST_POOLING_MODULE_TYPES = {
|
||||
"sentence_transformers.models.Pooling",
|
||||
"sentence_transformers.sentence_transformer.modules.pooling.Pooling",
|
||||
|
||||
@@ -14,6 +14,7 @@ from vllm.config.model_arch import (
|
||||
from vllm.config.utils import getattr_iter
|
||||
from vllm.logger import init_logger
|
||||
from vllm.transformers_utils.config import (
|
||||
CONFIG_FIELD_ALIASES,
|
||||
ConfigFormat,
|
||||
get_safetensors_params_metadata,
|
||||
)
|
||||
@@ -148,23 +149,11 @@ class ModelArchConfigConvertorBase:
|
||||
return qk_rope_head_dim
|
||||
|
||||
def get_total_num_kv_heads(self) -> int:
|
||||
attributes = [
|
||||
# For Falcon:
|
||||
"n_head_kv",
|
||||
"num_kv_heads",
|
||||
# For LLaMA-2:
|
||||
"num_key_value_heads",
|
||||
# For ChatGLM:
|
||||
"multi_query_group_num",
|
||||
# For Step3p5:
|
||||
"num_attention_groups",
|
||||
]
|
||||
names = CONFIG_FIELD_ALIASES["total_num_kv_heads"]
|
||||
# For non-grouped-query attention models, the number of KV heads is
|
||||
# equal to the number of attention heads.
|
||||
default_factory = self.get_total_num_attention_heads
|
||||
return getattr_iter(
|
||||
self.hf_text_config, attributes, default_factory=default_factory
|
||||
)
|
||||
return getattr_iter(self.hf_text_config, names, default_factory=default_factory)
|
||||
|
||||
def get_num_experts_from_block_configs(self) -> int:
|
||||
"""Check block_configs for heterogeneous models (e.g., NemotronH).
|
||||
@@ -188,13 +177,7 @@ class ModelArchConfigConvertorBase:
|
||||
|
||||
def get_num_experts(self) -> int:
|
||||
"""Returns the number of experts in the model."""
|
||||
num_expert_names = [
|
||||
"num_experts", # Jamba
|
||||
"moe_num_experts", # Dbrx
|
||||
"n_routed_experts", # DeepSeek
|
||||
"num_local_experts", # Mixtral
|
||||
]
|
||||
|
||||
num_expert_names = CONFIG_FIELD_ALIASES["num_experts"]
|
||||
num_experts = getattr_iter(self.hf_text_config, num_expert_names, 0)
|
||||
if isinstance(num_experts, list):
|
||||
# Ernie VL's remote code uses list[int]...
|
||||
@@ -206,13 +189,7 @@ class ModelArchConfigConvertorBase:
|
||||
return num_experts
|
||||
|
||||
def get_num_experts_per_token(self) -> int:
|
||||
names = [
|
||||
"num_experts_per_tok",
|
||||
"num_experts_per_token",
|
||||
"top_k_experts",
|
||||
"moe_topk",
|
||||
"moe_top_k",
|
||||
]
|
||||
names = CONFIG_FIELD_ALIASES["num_experts_per_token"]
|
||||
num_experts_per_token = getattr_iter(self.hf_text_config, names, 0)
|
||||
if isinstance(num_experts_per_token, list):
|
||||
return max(num_experts_per_token, default=0)
|
||||
|
||||
Reference in New Issue
Block a user