[MoE][Bugfix] Skip SP padded rows in grouped MoE routing (#56079)
Signed-off-by: Elvir Crncevic <elvircrn@gmail.com> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: Tyler Michael Smith <tlrmchlsmth@gmail.com>
This commit is contained in:
co-authored by
OpenAI Codex
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Tyler Michael Smith
parent
67e5b0acc9
commit
ac2f0ea82c
@@ -15,8 +15,10 @@ from vllm.config import (
|
||||
get_cached_compilation_config,
|
||||
set_current_vllm_config,
|
||||
)
|
||||
from vllm.forward_context import set_forward_context
|
||||
from vllm.model_executor.layers.fused_moe.router.grouped_topk_router import (
|
||||
GroupedTopk,
|
||||
GroupedTopKRouter,
|
||||
fused_grouped_topk,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
@@ -70,6 +72,36 @@ def _single_group_reference(
|
||||
return values, indices.to(torch.int32)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda(), reason="This test is skipped on non-CUDA platform."
|
||||
)
|
||||
@pytest.mark.parametrize("skip_padding", [False, True])
|
||||
def test_kimi_grouped_topk_masks_padding_when_enabled(skip_padding: bool):
|
||||
hidden_states = torch.randn((2, 16), device="cuda")
|
||||
gating_output = torch.randn((2, 896), device="cuda")
|
||||
is_padding = torch.tensor([False, True], device="cuda")
|
||||
router = GroupedTopKRouter(
|
||||
top_k=16,
|
||||
global_num_experts=896,
|
||||
num_expert_group=1,
|
||||
topk_group=1,
|
||||
skip_padding=skip_padding,
|
||||
)
|
||||
|
||||
with set_forward_context(None, VllmConfig(), is_padding=is_padding):
|
||||
topk_weights, topk_ids = router._compute_routing(
|
||||
hidden_states, gating_output, torch.int32
|
||||
)
|
||||
|
||||
assert torch.all(topk_ids[0] >= 0)
|
||||
if skip_padding:
|
||||
assert torch.all(topk_ids[1] == -1)
|
||||
assert torch.all(topk_weights[1] == 0)
|
||||
else:
|
||||
assert torch.all(topk_ids[1] >= 0)
|
||||
assert torch.all(topk_weights[1] > 0)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda(), reason="This test is skipped on non-CUDA platform."
|
||||
)
|
||||
|
||||
@@ -87,6 +87,21 @@ def test_multiple_expert_groups_use_grouped_topk() -> None:
|
||||
)
|
||||
|
||||
assert isinstance(router, GroupedTopKRouter)
|
||||
assert not router.skip_padding
|
||||
|
||||
|
||||
def test_grouped_topk_padding_skip_must_be_enabled() -> None:
|
||||
router = create_fused_moe_router(
|
||||
top_k=4,
|
||||
global_num_experts=128,
|
||||
use_grouped_topk=True,
|
||||
num_expert_group=8,
|
||||
topk_group=4,
|
||||
skip_padding=True,
|
||||
)
|
||||
|
||||
assert isinstance(router, GroupedTopKRouter)
|
||||
assert router.skip_padding
|
||||
|
||||
|
||||
def test_degenerate_grouped_config_with_bias_uses_topk_bias() -> None:
|
||||
|
||||
@@ -174,6 +174,20 @@ def test_freeze_gc_disables_and_restores_automatic_gc(
|
||||
gc.disable()
|
||||
|
||||
|
||||
def test_prepare_padding_mask_marks_sequence_parallel_padding():
|
||||
runner = GPUModelRunner.__new__(GPUModelRunner)
|
||||
runner.is_padding = torch.empty(8, dtype=torch.bool)
|
||||
|
||||
mask = runner._prepare_padding_mask(1, 8)
|
||||
|
||||
assert mask.tolist() == [False, True, True, True, True, True, True, True]
|
||||
assert mask.data_ptr() == runner.is_padding.data_ptr()
|
||||
|
||||
mask = runner._prepare_padding_mask(0, 8)
|
||||
|
||||
assert mask.all()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def model_runner():
|
||||
vllm_config = get_vllm_config()
|
||||
|
||||
@@ -21,6 +21,20 @@ from vllm.v1.worker.gpu.block_table import BlockTables
|
||||
from vllm.v1.worker.gpu.model_runner import GPUModelRunner
|
||||
|
||||
|
||||
def test_prepare_padding_mask_marks_sequence_parallel_padding():
|
||||
runner = GPUModelRunner.__new__(GPUModelRunner)
|
||||
runner.input_buffers = SimpleNamespace(is_padding=torch.empty(8, dtype=torch.bool))
|
||||
|
||||
mask = runner._prepare_padding_mask(1, 8)
|
||||
|
||||
assert mask.tolist() == [False, True, True, True, True, True, True, True]
|
||||
assert mask.data_ptr() == runner.input_buffers.is_padding.data_ptr()
|
||||
|
||||
mask = runner._prepare_padding_mask(0, 8)
|
||||
|
||||
assert mask.all()
|
||||
|
||||
|
||||
def test_qsa_circular_group_uses_custom_slot_mapping(monkeypatch):
|
||||
runner = GPUModelRunner.__new__(GPUModelRunner)
|
||||
runner.max_model_len = 262144
|
||||
|
||||
@@ -10,6 +10,7 @@ threaded execution of the microbatches.
|
||||
"""
|
||||
|
||||
import threading
|
||||
from contextlib import nullcontext
|
||||
from dataclasses import replace
|
||||
from types import SimpleNamespace
|
||||
from typing import Any, cast
|
||||
@@ -30,6 +31,7 @@ from vllm.config import (
|
||||
from vllm.forward_context import create_forward_context
|
||||
from vllm.v1.attention.backends.utils import get_dcp_local_seq_lens
|
||||
from vllm.v1.kv_cache_interface import KVCacheConfig
|
||||
from vllm.v1.worker import gpu_ubatch_wrapper as legacy_gpu_ubatch_wrapper
|
||||
from vllm.v1.worker.gpu import cp_utils as gpu_cp_utils
|
||||
from vllm.v1.worker.gpu import cudagraph_utils, dp_utils
|
||||
from vllm.v1.worker.gpu.cudagraph_utils import (
|
||||
@@ -426,6 +428,94 @@ def test_slice_model_inputs_handles_mrope_positions():
|
||||
assert sliced["intermediate_tensors"] is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("cudagraph_runtime_mode", "execution_method"),
|
||||
[
|
||||
(CUDAGraphMode.FULL, "_capture_ubatches"),
|
||||
(CUDAGraphMode.NONE, "_run_ubatches"),
|
||||
],
|
||||
)
|
||||
def test_legacy_ubatch_contexts_slice_padding_mask(
|
||||
cudagraph_runtime_mode: CUDAGraphMode, execution_method: str
|
||||
):
|
||||
"""Replacement contexts preserve each microbatch's padding rows."""
|
||||
ubatch_slices = [
|
||||
UBatchSlice(slice(0, 2), slice(0, 3)),
|
||||
UBatchSlice(slice(2, 4), slice(3, 6)),
|
||||
]
|
||||
is_padding = torch.tensor([False, True, False, True, False, True])
|
||||
forward_context = SimpleNamespace(
|
||||
batch_descriptor=object(),
|
||||
ubatch_slices=ubatch_slices,
|
||||
cudagraph_runtime_mode=cudagraph_runtime_mode,
|
||||
attn_metadata=[{}, {}],
|
||||
slot_mapping={},
|
||||
dp_metadata=object(),
|
||||
is_padding=is_padding,
|
||||
)
|
||||
|
||||
wrapper = object.__new__(legacy_gpu_ubatch_wrapper.UBatchWrapper)
|
||||
wrapper.vllm_config = SimpleNamespace(
|
||||
parallel_config=SimpleNamespace(data_parallel_size=2)
|
||||
)
|
||||
wrapper.comm_stream = object()
|
||||
wrapper.ready_barrier = object()
|
||||
wrapper.cudagraphs = {}
|
||||
wrapper.sm_control = nullcontext()
|
||||
wrapper.runnable = object()
|
||||
|
||||
def make_contexts(*, forward_contexts, **kwargs):
|
||||
return [
|
||||
SimpleNamespace(id=i, forward_context=context)
|
||||
for i, context in enumerate(forward_contexts)
|
||||
]
|
||||
|
||||
def create_context(attn_metadata, vllm_config, **kwargs):
|
||||
return SimpleNamespace(attn_metadata=attn_metadata, **kwargs)
|
||||
|
||||
model_inputs = {
|
||||
"input_ids": torch.arange(6),
|
||||
"positions": torch.arange(6),
|
||||
"inputs_embeds": None,
|
||||
"intermediate_tensors": None,
|
||||
}
|
||||
expected = object()
|
||||
with (
|
||||
patch.object(
|
||||
legacy_gpu_ubatch_wrapper,
|
||||
"get_forward_context",
|
||||
return_value=forward_context,
|
||||
),
|
||||
patch.object(
|
||||
legacy_gpu_ubatch_wrapper.torch.cuda,
|
||||
"current_stream",
|
||||
return_value=object(),
|
||||
),
|
||||
patch.object(
|
||||
legacy_gpu_ubatch_wrapper.DPMetadata, "make", return_value=object()
|
||||
),
|
||||
patch.object(
|
||||
legacy_gpu_ubatch_wrapper,
|
||||
"create_forward_context",
|
||||
side_effect=create_context,
|
||||
),
|
||||
patch.object(
|
||||
legacy_gpu_ubatch_wrapper,
|
||||
"make_ubatch_contexts",
|
||||
side_effect=make_contexts,
|
||||
),
|
||||
patch.object(wrapper, execution_method, return_value=expected) as execute,
|
||||
):
|
||||
result = wrapper(**model_inputs)
|
||||
|
||||
assert result is expected
|
||||
contexts = [
|
||||
metadata.context.forward_context for metadata in execute.call_args.args[0]
|
||||
]
|
||||
torch.testing.assert_close(contexts[0].is_padding, is_padding[:3])
|
||||
torch.testing.assert_close(contexts[1].is_padding, is_padding[3:])
|
||||
|
||||
|
||||
def _make_dbo_config() -> VllmConfig:
|
||||
return VllmConfig(
|
||||
model_config=ModelConfig(model="facebook/opt-125m", dtype="float16", seed=0),
|
||||
|
||||
Reference in New Issue
Block a user