[Transformers backend] Fuse MoE routers that score after the top-k (#57410)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Harry Mellor
2026-10-13 08:41:23 +00:00
committed by GitHub
co-authored by Claude Opus 5
parent 24aa1191f4
commit 6f11c220b5
6 changed files with 212 additions and 59 deletions
+18 -13
View File
@@ -39,6 +39,7 @@ from vllm.model_executor.models.transformers.fuser import get_fuser
from vllm.model_executor.models.transformers.fusers.glu import GLUFuser
from vllm.model_executor.models.transformers.fusers.moe import MoEBlockFuser
from vllm.model_executor.models.utils import extract_layer_index, maybe_prefix
from vllm.transformers_utils.config import CONFIG_FIELD_ALIASES
from vllm.utils.torch_utils import STR_DTYPE_TO_TORCH_DTYPE, direct_register_custom_op
from .base import Base
@@ -180,23 +181,18 @@ class MoEMixin(MixtureOfExperts, Base):
text_config = self.text_config
# Positional arguments
num_experts = self.model_config.get_num_experts()
top_k = getattr_iter(text_config, ["num_experts_per_tok", "top_k"], None)
num_experts_names = CONFIG_FIELD_ALIASES["num_experts"]
num_experts = getattr_iter(text_config, num_experts_names, None)
top_k_names = CONFIG_FIELD_ALIASES["num_experts_per_token"]
top_k = getattr_iter(text_config, top_k_names, None)
assert top_k is not None
hidden_size = text_config.hidden_size
intermediate_size = getattr_iter(
text_config, ["moe_intermediate_size", "intermediate_size"], None
)
intermediate_size_names = CONFIG_FIELD_ALIASES["moe_intermediate_size"]
intermediate_size = getattr_iter(text_config, intermediate_size_names, None)
assert intermediate_size is not None
num_shared_experts = getattr_iter(
text_config,
[
"n_shared_experts", # DeepSeek, Docs, GLM
"moe_num_shared_experts", # Aria, Ernie
],
0,
)
num_shared_experts_names = CONFIG_FIELD_ALIASES["num_shared_experts"]
num_shared_experts = getattr_iter(text_config, num_shared_experts_names, 0)
# Common kwargs
norm_topk_prob = getattr(text_config, "norm_topk_prob", None)
@@ -275,6 +271,9 @@ class MoEMixin(MixtureOfExperts, Base):
if "bias" in experts_param_name:
has_bias = True
break
# Whether the expert weights are stored in transposed format.
# Comes from Transformers `use_experts_implementation` decorator.
is_transposed = getattr(experts, "is_transposed", False)
# If the config does not specify num_shared_experts, but
# the model has shared experts, we assume there is one.
if self.num_shared_experts == 0:
@@ -308,6 +307,7 @@ class MoEMixin(MixtureOfExperts, Base):
enable_eplb=enable_eplb,
num_redundant_experts=num_redundant_experts,
has_bias=has_bias,
is_fused_checkpoint_transposed=is_transposed,
routed_experts_cls=TransformersRoutedExperts,
)
fuser = MoEBlockFuser.match(moe_block, experts_name)
@@ -346,6 +346,11 @@ class MoEMixin(MixtureOfExperts, Base):
gate = fuser.gate(moe_block, prefix, router_dtype)
kwargs |= dict(
scoring_func=fuser.scoring_func,
renormalize=(
layer_renormalize
if fuser.renormalize is None
else fuser.renormalize
),
is_sequence_parallel=(
self.parallel_config.use_sequence_parallel_moe
),