[ROCm][Perf] Enable DPA+ETP prefill compute/comm dual-batch-overlap for DeepSeek-V4 (#57773)
Signed-off-by: Shanshan Shen <87969357+shen-shanshan@users.noreply.github.com> Signed-off-by: shen-shanshan <467638484@qq.com>
This commit is contained in:
@@ -0,0 +1,118 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.layers.fused_moe.prepare_finalize import naive_dp_ep
|
||||
from vllm.model_executor.layers.fused_moe.prepare_finalize.naive_dp_ep import (
|
||||
MoEPrepareAndFinalizeNaiveDPEPModular,
|
||||
make_moe_prepare_and_finalize_naive_dp_ep,
|
||||
)
|
||||
|
||||
|
||||
class _RegionProbe(MoEPrepareAndFinalizeNaiveDPEPModular):
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
self.body_ran = False
|
||||
|
||||
def run_comm(self) -> None:
|
||||
with self._comm_region():
|
||||
self.body_ran = True
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def dbo_mocks():
|
||||
yield_switch = MagicMock()
|
||||
switch_back = MagicMock()
|
||||
with (
|
||||
patch.object(
|
||||
naive_dp_ep, "dbo_yield_and_switch_from_compute_to_comm", yield_switch
|
||||
),
|
||||
patch.object(naive_dp_ep, "dbo_switch_to_compute_sync", switch_back),
|
||||
):
|
||||
yield {"yield_switch": yield_switch, "switch_back": switch_back}
|
||||
|
||||
|
||||
def _dbo_state(dbo_active: bool, capturing: bool):
|
||||
return (
|
||||
patch.object(naive_dp_ep, "dbo_enabled", return_value=dbo_active),
|
||||
patch.object(torch.cuda, "is_current_stream_capturing", return_value=capturing),
|
||||
)
|
||||
|
||||
|
||||
def test_comm_region_noop_without_dbo(dbo_mocks):
|
||||
probe = _RegionProbe()
|
||||
dbo, capturing = _dbo_state(dbo_active=False, capturing=False)
|
||||
with dbo, capturing:
|
||||
probe.run_comm()
|
||||
assert probe.body_ran
|
||||
dbo_mocks["yield_switch"].assert_not_called()
|
||||
dbo_mocks["switch_back"].assert_not_called()
|
||||
|
||||
|
||||
def test_comm_region_handoff_when_dbo_active(dbo_mocks):
|
||||
probe = _RegionProbe()
|
||||
dbo, capturing = _dbo_state(dbo_active=True, capturing=False)
|
||||
with dbo, capturing:
|
||||
probe.run_comm()
|
||||
assert probe.body_ran
|
||||
dbo_mocks["yield_switch"].assert_called_once_with()
|
||||
dbo_mocks["switch_back"].assert_called_once_with()
|
||||
|
||||
|
||||
def test_comm_region_skipped_during_graph_capture(dbo_mocks):
|
||||
probe = _RegionProbe()
|
||||
dbo, capturing = _dbo_state(dbo_active=True, capturing=True)
|
||||
with dbo, capturing:
|
||||
probe.run_comm()
|
||||
assert probe.body_ran
|
||||
dbo_mocks["yield_switch"].assert_not_called()
|
||||
dbo_mocks["switch_back"].assert_not_called()
|
||||
|
||||
|
||||
def test_comm_region_restores_compute_stream_on_error(dbo_mocks):
|
||||
probe = _RegionProbe()
|
||||
dbo, capturing = _dbo_state(dbo_active=True, capturing=False)
|
||||
with (
|
||||
dbo,
|
||||
capturing,
|
||||
pytest.raises(RuntimeError, match="collective failed"),
|
||||
probe._comm_region(),
|
||||
):
|
||||
raise RuntimeError("collective failed")
|
||||
dbo_mocks["yield_switch"].assert_called_once_with()
|
||||
dbo_mocks["switch_back"].assert_called_once_with()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("is_rocm", [True, False])
|
||||
def test_supports_dbo_only_on_rocm(is_rocm):
|
||||
inst = MoEPrepareAndFinalizeNaiveDPEPModular()
|
||||
platform = SimpleNamespace(is_rocm=lambda: is_rocm)
|
||||
with patch.object(naive_dp_ep, "current_platform", platform):
|
||||
assert inst.supports_dbo() is is_rocm
|
||||
# supports_async() also drives MK-internal shared-expert overlap, so
|
||||
# it must not change with DBO support.
|
||||
assert not inst.supports_async()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("is_rocm", [True, False])
|
||||
def test_factory_returns_base_classes(is_rocm):
|
||||
platform = SimpleNamespace(is_rocm=lambda: is_rocm)
|
||||
with patch.object(naive_dp_ep, "current_platform", platform):
|
||||
mono = make_moe_prepare_and_finalize_naive_dp_ep(use_monolithic=True)
|
||||
modular = make_moe_prepare_and_finalize_naive_dp_ep(use_monolithic=False)
|
||||
assert type(mono) is naive_dp_ep.MoEPrepareAndFinalizeNaiveDPEPMonolithic
|
||||
assert type(modular) is MoEPrepareAndFinalizeNaiveDPEPModular
|
||||
|
||||
|
||||
def test_default_supports_dbo_follows_supports_async():
|
||||
# Classes that do not override supports_dbo() keep the old contract:
|
||||
# DBO only with async prepare/finalize.
|
||||
mono = naive_dp_ep.MoEPrepareAndFinalizeNaiveDPEPMonolithic()
|
||||
assert not mono.supports_async()
|
||||
assert not mono.supports_dbo()
|
||||
with patch.object(type(mono), "supports_async", return_value=True):
|
||||
assert mono.supports_dbo()
|
||||
@@ -0,0 +1,80 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import threading
|
||||
from unittest.mock import patch
|
||||
|
||||
import numpy as np
|
||||
|
||||
from vllm.v1.worker import ubatching
|
||||
from vllm.v1.worker.ubatch_utils import maybe_create_ubatch_slices
|
||||
from vllm.v1.worker.ubatching import (
|
||||
dbo_current_ubatch_token_offset,
|
||||
make_ubatch_contexts,
|
||||
)
|
||||
|
||||
|
||||
def test_token_offset_without_dbo_context():
|
||||
assert dbo_current_ubatch_token_offset() == 0
|
||||
|
||||
|
||||
def test_token_offset_defaults_to_zero():
|
||||
ready_barrier = threading.Barrier(3)
|
||||
ctxs = make_ubatch_contexts(
|
||||
num_micro_batches=2,
|
||||
compute_stream=object(),
|
||||
comm_stream=object(),
|
||||
forward_contexts=[object(), object()],
|
||||
ready_barrier=ready_barrier,
|
||||
)
|
||||
assert [ctx.token_offset for ctx in ctxs] == [0, 0]
|
||||
|
||||
|
||||
def test_token_offset_per_ubatch_thread():
|
||||
compute_stream = object()
|
||||
ready_barrier = threading.Barrier(3)
|
||||
ctxs = make_ubatch_contexts(
|
||||
num_micro_batches=2,
|
||||
compute_stream=compute_stream,
|
||||
comm_stream=object(),
|
||||
forward_contexts=[object(), object()],
|
||||
ready_barrier=ready_barrier,
|
||||
token_offsets=[0, 7],
|
||||
)
|
||||
assert [ctx.token_offset for ctx in ctxs] == [0, 7]
|
||||
|
||||
offsets_seen: dict[int, int] = {}
|
||||
|
||||
def run(ctx: ubatching.UBatchContext) -> None:
|
||||
with ctx:
|
||||
offsets_seen[ctx.id] = dbo_current_ubatch_token_offset()
|
||||
|
||||
# Keep torch.cuda.set_stream out of the way. Patch once here rather than
|
||||
# per thread: a per-thread patch exit would restore the real function
|
||||
# while the other thread is still inside its context.
|
||||
with patch.object(ubatching, "current_stream", return_value=compute_stream):
|
||||
threads = [threading.Thread(target=run, args=(ctx,)) for ctx in ctxs]
|
||||
for thread in threads:
|
||||
thread.start()
|
||||
ready_barrier.wait()
|
||||
ctxs[0].cpu_wait_event.set()
|
||||
for thread in threads:
|
||||
thread.join()
|
||||
|
||||
assert offsets_seen == {0: 0, 1: 7}
|
||||
|
||||
|
||||
def test_ubatch_slice_bounds_are_python_ints():
|
||||
# The last bound comes from np.cumsum; numpy scalars leaking into the
|
||||
# per-ubatch metadata break Triton argument specialization.
|
||||
ubatch_slices, ubatch_slices_padded = maybe_create_ubatch_slices(
|
||||
should_ubatch=True,
|
||||
num_scheduled_tokens=np.array([5, 9, 6], dtype=np.int32),
|
||||
num_tokens_padded=20,
|
||||
num_reqs_padded=3,
|
||||
num_ubatches=2,
|
||||
)
|
||||
for slices in (ubatch_slices, ubatch_slices_padded):
|
||||
assert slices is not None
|
||||
for s in slices:
|
||||
for bound in (s.token_slice.start, s.token_slice.stop):
|
||||
assert type(bound) is int
|
||||
Reference in New Issue
Block a user