[Transformers backend] Fuse MoE routers that score after the top-k (#57410)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Harry Mellor
2026-10-13 08:41:23 +00:00
committed by GitHub
co-authored by Claude Opus 5
parent 24aa1191f4
commit 6f11c220b5
6 changed files with 212 additions and 59 deletions
+36
View File
@@ -52,6 +52,42 @@ MISTRAL_CONFIG_NAME = "params.json"
logger = init_logger(__name__)
CONFIG_FIELD_ALIASES: dict[str, tuple[str, ...]] = {
"total_num_kv_heads": (
"n_head_kv", # Falcon
"num_kv_heads",
"num_key_value_heads", # LLaMA-2
"multi_query_group_num", # ChatGLM
"num_attention_groups", # Step3p5
),
"num_experts": (
"num_experts", # Jamba
"moe_num_experts", # Dbrx
"n_routed_experts", # DeepSeek
"num_local_experts", # Mixtral
),
"num_experts_per_token": (
"num_experts_per_tok",
"num_experts_per_token",
"top_k_experts",
"moe_topk", # Hunyuan
"moe_top_k",
"top_k",
),
"moe_intermediate_size": (
"moe_intermediate_size",
"intermediate_size",
),
"num_shared_experts": (
"n_shared_experts", # DeepSeek, Docs, GLM
"moe_num_shared_experts", # Aria, Ernie
),
}
"""HF config fields that models spell differently, keyed by the name vLLM uses.
`ModelArchConfigConvertorBase` reads a scalar from these; a caller that needs the
raw value (a per-layer `list`, say) passes the aliases to `getattr_iter` itself."""
_ST_POOLING_MODULE_TYPES = {
"sentence_transformers.models.Pooling",
"sentence_transformers.sentence_transformer.modules.pooling.Pooling",
@@ -14,6 +14,7 @@ from vllm.config.model_arch import (
from vllm.config.utils import getattr_iter
from vllm.logger import init_logger
from vllm.transformers_utils.config import (
CONFIG_FIELD_ALIASES,
ConfigFormat,
get_safetensors_params_metadata,
)
@@ -148,23 +149,11 @@ class ModelArchConfigConvertorBase:
return qk_rope_head_dim
def get_total_num_kv_heads(self) -> int:
attributes = [
# For Falcon:
"n_head_kv",
"num_kv_heads",
# For LLaMA-2:
"num_key_value_heads",
# For ChatGLM:
"multi_query_group_num",
# For Step3p5:
"num_attention_groups",
]
names = CONFIG_FIELD_ALIASES["total_num_kv_heads"]
# For non-grouped-query attention models, the number of KV heads is
# equal to the number of attention heads.
default_factory = self.get_total_num_attention_heads
return getattr_iter(
self.hf_text_config, attributes, default_factory=default_factory
)
return getattr_iter(self.hf_text_config, names, default_factory=default_factory)
def get_num_experts_from_block_configs(self) -> int:
"""Check block_configs for heterogeneous models (e.g., NemotronH).
@@ -188,13 +177,7 @@ class ModelArchConfigConvertorBase:
def get_num_experts(self) -> int:
"""Returns the number of experts in the model."""
num_expert_names = [
"num_experts", # Jamba
"moe_num_experts", # Dbrx
"n_routed_experts", # DeepSeek
"num_local_experts", # Mixtral
]
num_expert_names = CONFIG_FIELD_ALIASES["num_experts"]
num_experts = getattr_iter(self.hf_text_config, num_expert_names, 0)
if isinstance(num_experts, list):
# Ernie VL's remote code uses list[int]...
@@ -206,13 +189,7 @@ class ModelArchConfigConvertorBase:
return num_experts
def get_num_experts_per_token(self) -> int:
names = [
"num_experts_per_tok",
"num_experts_per_token",
"top_k_experts",
"moe_topk",
"moe_top_k",
]
names = CONFIG_FIELD_ALIASES["num_experts_per_token"]
num_experts_per_token = getattr_iter(self.hf_text_config, names, 0)
if isinstance(num_experts_per_token, list):
return max(num_experts_per_token, default=0)