[ROCm][Perf] Enable DPA+ETP prefill compute/comm dual-batch-overlap for DeepSeek-V4 (#57773)

Signed-off-by: Shanshan Shen <87969357+shen-shanshan@users.noreply.github.com>
Signed-off-by: shen-shanshan <467638484@qq.com>
This commit is contained in:
Shanshan Shen
2026-10-15 11:16:48 +08:00
committed by GitHub
parent 5d5c3d7c43
commit f46021a7fa
9 changed files with 319 additions and 28 deletions
+118
View File
@@ -0,0 +1,118 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from types import SimpleNamespace
from unittest.mock import MagicMock, patch
import pytest
import torch
from vllm.model_executor.layers.fused_moe.prepare_finalize import naive_dp_ep
from vllm.model_executor.layers.fused_moe.prepare_finalize.naive_dp_ep import (
MoEPrepareAndFinalizeNaiveDPEPModular,
make_moe_prepare_and_finalize_naive_dp_ep,
)
class _RegionProbe(MoEPrepareAndFinalizeNaiveDPEPModular):
def __init__(self) -> None:
super().__init__()
self.body_ran = False
def run_comm(self) -> None:
with self._comm_region():
self.body_ran = True
@pytest.fixture
def dbo_mocks():
yield_switch = MagicMock()
switch_back = MagicMock()
with (
patch.object(
naive_dp_ep, "dbo_yield_and_switch_from_compute_to_comm", yield_switch
),
patch.object(naive_dp_ep, "dbo_switch_to_compute_sync", switch_back),
):
yield {"yield_switch": yield_switch, "switch_back": switch_back}
def _dbo_state(dbo_active: bool, capturing: bool):
return (
patch.object(naive_dp_ep, "dbo_enabled", return_value=dbo_active),
patch.object(torch.cuda, "is_current_stream_capturing", return_value=capturing),
)
def test_comm_region_noop_without_dbo(dbo_mocks):
probe = _RegionProbe()
dbo, capturing = _dbo_state(dbo_active=False, capturing=False)
with dbo, capturing:
probe.run_comm()
assert probe.body_ran
dbo_mocks["yield_switch"].assert_not_called()
dbo_mocks["switch_back"].assert_not_called()
def test_comm_region_handoff_when_dbo_active(dbo_mocks):
probe = _RegionProbe()
dbo, capturing = _dbo_state(dbo_active=True, capturing=False)
with dbo, capturing:
probe.run_comm()
assert probe.body_ran
dbo_mocks["yield_switch"].assert_called_once_with()
dbo_mocks["switch_back"].assert_called_once_with()
def test_comm_region_skipped_during_graph_capture(dbo_mocks):
probe = _RegionProbe()
dbo, capturing = _dbo_state(dbo_active=True, capturing=True)
with dbo, capturing:
probe.run_comm()
assert probe.body_ran
dbo_mocks["yield_switch"].assert_not_called()
dbo_mocks["switch_back"].assert_not_called()
def test_comm_region_restores_compute_stream_on_error(dbo_mocks):
probe = _RegionProbe()
dbo, capturing = _dbo_state(dbo_active=True, capturing=False)
with (
dbo,
capturing,
pytest.raises(RuntimeError, match="collective failed"),
probe._comm_region(),
):
raise RuntimeError("collective failed")
dbo_mocks["yield_switch"].assert_called_once_with()
dbo_mocks["switch_back"].assert_called_once_with()
@pytest.mark.parametrize("is_rocm", [True, False])
def test_supports_dbo_only_on_rocm(is_rocm):
inst = MoEPrepareAndFinalizeNaiveDPEPModular()
platform = SimpleNamespace(is_rocm=lambda: is_rocm)
with patch.object(naive_dp_ep, "current_platform", platform):
assert inst.supports_dbo() is is_rocm
# supports_async() also drives MK-internal shared-expert overlap, so
# it must not change with DBO support.
assert not inst.supports_async()
@pytest.mark.parametrize("is_rocm", [True, False])
def test_factory_returns_base_classes(is_rocm):
platform = SimpleNamespace(is_rocm=lambda: is_rocm)
with patch.object(naive_dp_ep, "current_platform", platform):
mono = make_moe_prepare_and_finalize_naive_dp_ep(use_monolithic=True)
modular = make_moe_prepare_and_finalize_naive_dp_ep(use_monolithic=False)
assert type(mono) is naive_dp_ep.MoEPrepareAndFinalizeNaiveDPEPMonolithic
assert type(modular) is MoEPrepareAndFinalizeNaiveDPEPModular
def test_default_supports_dbo_follows_supports_async():
# Classes that do not override supports_dbo() keep the old contract:
# DBO only with async prepare/finalize.
mono = naive_dp_ep.MoEPrepareAndFinalizeNaiveDPEPMonolithic()
assert not mono.supports_async()
assert not mono.supports_dbo()
with patch.object(type(mono), "supports_async", return_value=True):
assert mono.supports_dbo()
+80
View File
@@ -0,0 +1,80 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import threading
from unittest.mock import patch
import numpy as np
from vllm.v1.worker import ubatching
from vllm.v1.worker.ubatch_utils import maybe_create_ubatch_slices
from vllm.v1.worker.ubatching import (
dbo_current_ubatch_token_offset,
make_ubatch_contexts,
)
def test_token_offset_without_dbo_context():
assert dbo_current_ubatch_token_offset() == 0
def test_token_offset_defaults_to_zero():
ready_barrier = threading.Barrier(3)
ctxs = make_ubatch_contexts(
num_micro_batches=2,
compute_stream=object(),
comm_stream=object(),
forward_contexts=[object(), object()],
ready_barrier=ready_barrier,
)
assert [ctx.token_offset for ctx in ctxs] == [0, 0]
def test_token_offset_per_ubatch_thread():
compute_stream = object()
ready_barrier = threading.Barrier(3)
ctxs = make_ubatch_contexts(
num_micro_batches=2,
compute_stream=compute_stream,
comm_stream=object(),
forward_contexts=[object(), object()],
ready_barrier=ready_barrier,
token_offsets=[0, 7],
)
assert [ctx.token_offset for ctx in ctxs] == [0, 7]
offsets_seen: dict[int, int] = {}
def run(ctx: ubatching.UBatchContext) -> None:
with ctx:
offsets_seen[ctx.id] = dbo_current_ubatch_token_offset()
# Keep torch.cuda.set_stream out of the way. Patch once here rather than
# per thread: a per-thread patch exit would restore the real function
# while the other thread is still inside its context.
with patch.object(ubatching, "current_stream", return_value=compute_stream):
threads = [threading.Thread(target=run, args=(ctx,)) for ctx in ctxs]
for thread in threads:
thread.start()
ready_barrier.wait()
ctxs[0].cpu_wait_event.set()
for thread in threads:
thread.join()
assert offsets_seen == {0: 0, 1: 7}
def test_ubatch_slice_bounds_are_python_ints():
# The last bound comes from np.cumsum; numpy scalars leaking into the
# per-ubatch metadata break Triton argument specialization.
ubatch_slices, ubatch_slices_padded = maybe_create_ubatch_slices(
should_ubatch=True,
num_scheduled_tokens=np.array([5, 9, 6], dtype=np.int32),
num_tokens_padded=20,
num_reqs_padded=3,
num_ubatches=2,
)
for slices in (ubatch_slices, ubatch_slices_padded):
assert slices is not None
for s in slices:
for bound in (s.token_slice.start, s.token_slice.stop):
assert type(bound) is int