[MoE][Bugfix] Skip SP padded rows in grouped MoE routing (#56079)
Signed-off-by: Elvir Crncevic <elvircrn@gmail.com> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: Tyler Michael Smith <tlrmchlsmth@gmail.com>
This commit is contained in:
co-authored by
OpenAI Codex
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Tyler Michael Smith
parent
67e5b0acc9
commit
ac2f0ea82c
@@ -15,8 +15,10 @@ from vllm.config import (
|
||||
get_cached_compilation_config,
|
||||
set_current_vllm_config,
|
||||
)
|
||||
from vllm.forward_context import set_forward_context
|
||||
from vllm.model_executor.layers.fused_moe.router.grouped_topk_router import (
|
||||
GroupedTopk,
|
||||
GroupedTopKRouter,
|
||||
fused_grouped_topk,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
@@ -70,6 +72,36 @@ def _single_group_reference(
|
||||
return values, indices.to(torch.int32)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda(), reason="This test is skipped on non-CUDA platform."
|
||||
)
|
||||
@pytest.mark.parametrize("skip_padding", [False, True])
|
||||
def test_kimi_grouped_topk_masks_padding_when_enabled(skip_padding: bool):
|
||||
hidden_states = torch.randn((2, 16), device="cuda")
|
||||
gating_output = torch.randn((2, 896), device="cuda")
|
||||
is_padding = torch.tensor([False, True], device="cuda")
|
||||
router = GroupedTopKRouter(
|
||||
top_k=16,
|
||||
global_num_experts=896,
|
||||
num_expert_group=1,
|
||||
topk_group=1,
|
||||
skip_padding=skip_padding,
|
||||
)
|
||||
|
||||
with set_forward_context(None, VllmConfig(), is_padding=is_padding):
|
||||
topk_weights, topk_ids = router._compute_routing(
|
||||
hidden_states, gating_output, torch.int32
|
||||
)
|
||||
|
||||
assert torch.all(topk_ids[0] >= 0)
|
||||
if skip_padding:
|
||||
assert torch.all(topk_ids[1] == -1)
|
||||
assert torch.all(topk_weights[1] == 0)
|
||||
else:
|
||||
assert torch.all(topk_ids[1] >= 0)
|
||||
assert torch.all(topk_weights[1] > 0)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda(), reason="This test is skipped on non-CUDA platform."
|
||||
)
|
||||
|
||||
@@ -87,6 +87,21 @@ def test_multiple_expert_groups_use_grouped_topk() -> None:
|
||||
)
|
||||
|
||||
assert isinstance(router, GroupedTopKRouter)
|
||||
assert not router.skip_padding
|
||||
|
||||
|
||||
def test_grouped_topk_padding_skip_must_be_enabled() -> None:
|
||||
router = create_fused_moe_router(
|
||||
top_k=4,
|
||||
global_num_experts=128,
|
||||
use_grouped_topk=True,
|
||||
num_expert_group=8,
|
||||
topk_group=4,
|
||||
skip_padding=True,
|
||||
)
|
||||
|
||||
assert isinstance(router, GroupedTopKRouter)
|
||||
assert router.skip_padding
|
||||
|
||||
|
||||
def test_degenerate_grouped_config_with_bias_uses_topk_bias() -> None:
|
||||
|
||||
Reference in New Issue
Block a user