[R3] Unify routed expert shape configuration (#50940)
Signed-off-by: aoshen02 <aoshen@inferact.ai> Co-authored-by: yihengz <yihengz@nvidia.com>
This commit is contained in:
@@ -4,6 +4,8 @@
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from typing import cast
|
||||
|
||||
import pytest
|
||||
from transformers import PretrainedConfig
|
||||
@@ -131,6 +133,38 @@ def test_head_size_falls_back_when_head_dim_is_zero():
|
||||
assert convertor.get_head_size() == 128
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"attribute",
|
||||
[
|
||||
"num_experts_per_tok",
|
||||
"num_experts_per_token",
|
||||
"top_k_experts",
|
||||
"moe_topk",
|
||||
"moe_top_k",
|
||||
],
|
||||
)
|
||||
def test_num_experts_per_tok_aliases(attribute: str):
|
||||
hf_config = PretrainedConfig(**{attribute: 4})
|
||||
convertor = ModelArchConfigConvertorBase(hf_config, hf_config)
|
||||
model_config = cast(
|
||||
ModelConfig,
|
||||
SimpleNamespace(
|
||||
model_arch_config=SimpleNamespace(
|
||||
num_experts_per_token=convertor.get_num_experts_per_token()
|
||||
)
|
||||
),
|
||||
)
|
||||
|
||||
assert ModelConfig.get_num_experts_per_tok(model_config) == 4
|
||||
|
||||
|
||||
def test_num_experts_per_tok_none_is_normalized():
|
||||
hf_config = PretrainedConfig(top_k_experts=None)
|
||||
convertor = ModelArchConfigConvertorBase(hf_config, hf_config)
|
||||
|
||||
assert convertor.get_num_experts_per_token() == 0
|
||||
|
||||
|
||||
def test_legacy_modelopt_config_without_producer_is_normalized():
|
||||
quantization_config = {
|
||||
"quantization": {
|
||||
|
||||
@@ -7,16 +7,25 @@ from unittest.mock import Mock, patch
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.config import ModelConfig, VllmConfig
|
||||
from vllm.config.compilation import CompilationMode
|
||||
from vllm.distributed.eplb.eplb_state import EplbLayerState
|
||||
from vllm.model_executor.layers.fused_moe.config import RoutingMethodType
|
||||
from vllm.model_executor.layers.fused_moe.routed_experts_capturer import (
|
||||
RoutedExpertsCapturer,
|
||||
RoutedExpertsManager,
|
||||
bind_routed_experts_capturer,
|
||||
get_routed_experts_attn_gid,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.router.base_router import BaseRouter
|
||||
from vllm.transformers_utils.model_arch_config_convertor import (
|
||||
ModelArchConfigConvertorBase,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
FullAttentionSpec,
|
||||
KVCacheConfig,
|
||||
KVCacheGroupSpec,
|
||||
)
|
||||
|
||||
pytestmark = pytest.mark.cpu_test
|
||||
|
||||
@@ -76,6 +85,72 @@ def _make_modular_routed_experts():
|
||||
)
|
||||
|
||||
|
||||
def _make_model_config(hf_config):
|
||||
num_experts_per_token = ModelArchConfigConvertorBase(
|
||||
hf_config, hf_config
|
||||
).get_num_experts_per_token()
|
||||
model_config = SimpleNamespace(
|
||||
hf_text_config=hf_config,
|
||||
model_arch_config=SimpleNamespace(
|
||||
num_experts_per_token=num_experts_per_token,
|
||||
),
|
||||
)
|
||||
model_config.get_num_experts = lambda: hf_config.num_experts
|
||||
model_config.get_num_experts_per_tok = lambda: (
|
||||
ModelConfig.get_num_experts_per_tok(model_config)
|
||||
)
|
||||
model_config.get_total_num_hidden_layers = lambda: hf_config.num_hidden_layers
|
||||
return model_config
|
||||
|
||||
|
||||
def test_routed_experts_manager_uses_gemma4_top_k_experts():
|
||||
hf_config = SimpleNamespace(
|
||||
num_experts=8,
|
||||
top_k_experts=2,
|
||||
num_hidden_layers=3,
|
||||
)
|
||||
vllm_config = SimpleNamespace(model_config=_make_model_config(hf_config))
|
||||
kv_cache_spec = FullAttentionSpec(
|
||||
block_size=4,
|
||||
num_kv_heads=1,
|
||||
head_size=1,
|
||||
dtype=torch.float32,
|
||||
)
|
||||
kv_cache_config = KVCacheConfig(
|
||||
num_blocks=2,
|
||||
kv_cache_tensors=[],
|
||||
kv_cache_groups=[KVCacheGroupSpec(["layer"], kv_cache_spec)],
|
||||
)
|
||||
|
||||
manager = RoutedExpertsManager(vllm_config, kv_cache_config)
|
||||
|
||||
assert manager.routed_experts_by_slot.shape == (8, 3, 2)
|
||||
|
||||
|
||||
def test_routed_experts_manager_uses_kimi_k3_experts_per_token():
|
||||
hf_config = SimpleNamespace(
|
||||
num_experts=8,
|
||||
num_experts_per_token=2,
|
||||
num_hidden_layers=3,
|
||||
)
|
||||
vllm_config = SimpleNamespace(model_config=_make_model_config(hf_config))
|
||||
kv_cache_spec = FullAttentionSpec(
|
||||
block_size=4,
|
||||
num_kv_heads=1,
|
||||
head_size=1,
|
||||
dtype=torch.float32,
|
||||
)
|
||||
kv_cache_config = KVCacheConfig(
|
||||
num_blocks=2,
|
||||
kv_cache_tensors=[],
|
||||
kv_cache_groups=[KVCacheGroupSpec(["layer"], kv_cache_spec)],
|
||||
)
|
||||
|
||||
manager = RoutedExpertsManager(vllm_config, kv_cache_config)
|
||||
|
||||
assert manager.routed_experts_by_slot.shape == (8, 3, 2)
|
||||
|
||||
|
||||
def test_base_router_capture_pre_eplb_mapping():
|
||||
router = _make_router()
|
||||
captured = []
|
||||
|
||||
Reference in New Issue
Block a user