[Transformers backend] Fuse MoE routers that score after the top-k (#57410)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5
parent
24aa1191f4
commit
6f11c220b5
@@ -52,6 +52,42 @@ MISTRAL_CONFIG_NAME = "params.json"
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
CONFIG_FIELD_ALIASES: dict[str, tuple[str, ...]] = {
|
||||
"total_num_kv_heads": (
|
||||
"n_head_kv", # Falcon
|
||||
"num_kv_heads",
|
||||
"num_key_value_heads", # LLaMA-2
|
||||
"multi_query_group_num", # ChatGLM
|
||||
"num_attention_groups", # Step3p5
|
||||
),
|
||||
"num_experts": (
|
||||
"num_experts", # Jamba
|
||||
"moe_num_experts", # Dbrx
|
||||
"n_routed_experts", # DeepSeek
|
||||
"num_local_experts", # Mixtral
|
||||
),
|
||||
"num_experts_per_token": (
|
||||
"num_experts_per_tok",
|
||||
"num_experts_per_token",
|
||||
"top_k_experts",
|
||||
"moe_topk", # Hunyuan
|
||||
"moe_top_k",
|
||||
"top_k",
|
||||
),
|
||||
"moe_intermediate_size": (
|
||||
"moe_intermediate_size",
|
||||
"intermediate_size",
|
||||
),
|
||||
"num_shared_experts": (
|
||||
"n_shared_experts", # DeepSeek, Docs, GLM
|
||||
"moe_num_shared_experts", # Aria, Ernie
|
||||
),
|
||||
}
|
||||
"""HF config fields that models spell differently, keyed by the name vLLM uses.
|
||||
|
||||
`ModelArchConfigConvertorBase` reads a scalar from these; a caller that needs the
|
||||
raw value (a per-layer `list`, say) passes the aliases to `getattr_iter` itself."""
|
||||
|
||||
_ST_POOLING_MODULE_TYPES = {
|
||||
"sentence_transformers.models.Pooling",
|
||||
"sentence_transformers.sentence_transformer.modules.pooling.Pooling",
|
||||
|
||||
@@ -14,6 +14,7 @@ from vllm.config.model_arch import (
|
||||
from vllm.config.utils import getattr_iter
|
||||
from vllm.logger import init_logger
|
||||
from vllm.transformers_utils.config import (
|
||||
CONFIG_FIELD_ALIASES,
|
||||
ConfigFormat,
|
||||
get_safetensors_params_metadata,
|
||||
)
|
||||
@@ -148,23 +149,11 @@ class ModelArchConfigConvertorBase:
|
||||
return qk_rope_head_dim
|
||||
|
||||
def get_total_num_kv_heads(self) -> int:
|
||||
attributes = [
|
||||
# For Falcon:
|
||||
"n_head_kv",
|
||||
"num_kv_heads",
|
||||
# For LLaMA-2:
|
||||
"num_key_value_heads",
|
||||
# For ChatGLM:
|
||||
"multi_query_group_num",
|
||||
# For Step3p5:
|
||||
"num_attention_groups",
|
||||
]
|
||||
names = CONFIG_FIELD_ALIASES["total_num_kv_heads"]
|
||||
# For non-grouped-query attention models, the number of KV heads is
|
||||
# equal to the number of attention heads.
|
||||
default_factory = self.get_total_num_attention_heads
|
||||
return getattr_iter(
|
||||
self.hf_text_config, attributes, default_factory=default_factory
|
||||
)
|
||||
return getattr_iter(self.hf_text_config, names, default_factory=default_factory)
|
||||
|
||||
def get_num_experts_from_block_configs(self) -> int:
|
||||
"""Check block_configs for heterogeneous models (e.g., NemotronH).
|
||||
@@ -188,13 +177,7 @@ class ModelArchConfigConvertorBase:
|
||||
|
||||
def get_num_experts(self) -> int:
|
||||
"""Returns the number of experts in the model."""
|
||||
num_expert_names = [
|
||||
"num_experts", # Jamba
|
||||
"moe_num_experts", # Dbrx
|
||||
"n_routed_experts", # DeepSeek
|
||||
"num_local_experts", # Mixtral
|
||||
]
|
||||
|
||||
num_expert_names = CONFIG_FIELD_ALIASES["num_experts"]
|
||||
num_experts = getattr_iter(self.hf_text_config, num_expert_names, 0)
|
||||
if isinstance(num_experts, list):
|
||||
# Ernie VL's remote code uses list[int]...
|
||||
@@ -206,13 +189,7 @@ class ModelArchConfigConvertorBase:
|
||||
return num_experts
|
||||
|
||||
def get_num_experts_per_token(self) -> int:
|
||||
names = [
|
||||
"num_experts_per_tok",
|
||||
"num_experts_per_token",
|
||||
"top_k_experts",
|
||||
"moe_topk",
|
||||
"moe_top_k",
|
||||
]
|
||||
names = CONFIG_FIELD_ALIASES["num_experts_per_token"]
|
||||
num_experts_per_token = getattr_iter(self.hf_text_config, names, 0)
|
||||
if isinstance(num_experts_per_token, list):
|
||||
return max(num_experts_per_token, default=0)
|
||||
|
||||
Reference in New Issue
Block a user