[R3] Unify routed expert shape configuration (#50940)

Signed-off-by: aoshen02 <aoshen@inferact.ai>
Co-authored-by: yihengz <yihengz@nvidia.com>
This commit is contained in:
aoshen02
2026-08-04 20:20:41 -07:00
committed by GitHub
co-authored by yihengz
parent 0187f4c88e
commit 6153dbe036
7 changed files with 146 additions and 46 deletions
+34
View File
@@ -4,6 +4,8 @@
import json
from pathlib import Path
from types import SimpleNamespace
from typing import cast
import pytest
from transformers import PretrainedConfig
@@ -131,6 +133,38 @@ def test_head_size_falls_back_when_head_dim_is_zero():
assert convertor.get_head_size() == 128
@pytest.mark.parametrize(
"attribute",
[
"num_experts_per_tok",
"num_experts_per_token",
"top_k_experts",
"moe_topk",
"moe_top_k",
],
)
def test_num_experts_per_tok_aliases(attribute: str):
hf_config = PretrainedConfig(**{attribute: 4})
convertor = ModelArchConfigConvertorBase(hf_config, hf_config)
model_config = cast(
ModelConfig,
SimpleNamespace(
model_arch_config=SimpleNamespace(
num_experts_per_token=convertor.get_num_experts_per_token()
)
),
)
assert ModelConfig.get_num_experts_per_tok(model_config) == 4
def test_num_experts_per_tok_none_is_normalized():
hf_config = PretrainedConfig(top_k_experts=None)
convertor = ModelArchConfigConvertorBase(hf_config, hf_config)
assert convertor.get_num_experts_per_token() == 0
def test_legacy_modelopt_config_without_producer_is_normalized():
quantization_config = {
"quantization": {
@@ -7,16 +7,25 @@ from unittest.mock import Mock, patch
import pytest
import torch
from vllm.config import VllmConfig
from vllm.config import ModelConfig, VllmConfig
from vllm.config.compilation import CompilationMode
from vllm.distributed.eplb.eplb_state import EplbLayerState
from vllm.model_executor.layers.fused_moe.config import RoutingMethodType
from vllm.model_executor.layers.fused_moe.routed_experts_capturer import (
RoutedExpertsCapturer,
RoutedExpertsManager,
bind_routed_experts_capturer,
get_routed_experts_attn_gid,
)
from vllm.model_executor.layers.fused_moe.router.base_router import BaseRouter
from vllm.transformers_utils.model_arch_config_convertor import (
ModelArchConfigConvertorBase,
)
from vllm.v1.kv_cache_interface import (
FullAttentionSpec,
KVCacheConfig,
KVCacheGroupSpec,
)
pytestmark = pytest.mark.cpu_test
@@ -76,6 +85,72 @@ def _make_modular_routed_experts():
)
def _make_model_config(hf_config):
num_experts_per_token = ModelArchConfigConvertorBase(
hf_config, hf_config
).get_num_experts_per_token()
model_config = SimpleNamespace(
hf_text_config=hf_config,
model_arch_config=SimpleNamespace(
num_experts_per_token=num_experts_per_token,
),
)
model_config.get_num_experts = lambda: hf_config.num_experts
model_config.get_num_experts_per_tok = lambda: (
ModelConfig.get_num_experts_per_tok(model_config)
)
model_config.get_total_num_hidden_layers = lambda: hf_config.num_hidden_layers
return model_config
def test_routed_experts_manager_uses_gemma4_top_k_experts():
hf_config = SimpleNamespace(
num_experts=8,
top_k_experts=2,
num_hidden_layers=3,
)
vllm_config = SimpleNamespace(model_config=_make_model_config(hf_config))
kv_cache_spec = FullAttentionSpec(
block_size=4,
num_kv_heads=1,
head_size=1,
dtype=torch.float32,
)
kv_cache_config = KVCacheConfig(
num_blocks=2,
kv_cache_tensors=[],
kv_cache_groups=[KVCacheGroupSpec(["layer"], kv_cache_spec)],
)
manager = RoutedExpertsManager(vllm_config, kv_cache_config)
assert manager.routed_experts_by_slot.shape == (8, 3, 2)
def test_routed_experts_manager_uses_kimi_k3_experts_per_token():
hf_config = SimpleNamespace(
num_experts=8,
num_experts_per_token=2,
num_hidden_layers=3,
)
vllm_config = SimpleNamespace(model_config=_make_model_config(hf_config))
kv_cache_spec = FullAttentionSpec(
block_size=4,
num_kv_heads=1,
head_size=1,
dtype=torch.float32,
)
kv_cache_config = KVCacheConfig(
num_blocks=2,
kv_cache_tensors=[],
kv_cache_groups=[KVCacheGroupSpec(["layer"], kv_cache_spec)],
)
manager = RoutedExpertsManager(vllm_config, kv_cache_config)
assert manager.routed_experts_by_slot.shape == (8, 3, 2)
def test_base_router_capture_pre_eplb_mapping():
router = _make_router()
captured = []