[ROCm][Perf] Enable DPA+ETP prefill compute/comm dual-batch-overlap for DeepSeek-V4 (#57773)

Signed-off-by: Shanshan Shen <87969357+shen-shanshan@users.noreply.github.com>
Signed-off-by: shen-shanshan <467638484@qq.com>
This commit is contained in:
Shanshan Shen
2026-10-15 11:16:48 +08:00
committed by GitHub
parent 5d5c3d7c43
commit f46021a7fa
9 changed files with 319 additions and 28 deletions
@@ -258,6 +258,15 @@ class FusedMoEPrepareAndFinalize(ABC):
"""
return False
def supports_dbo(self) -> bool:
"""Indicates whether or not this class can run under DBO.
Defaults to supports_async(). A class whose synchronous prepare and
finalize do their own DBO compute/comm handoff can return True here
while keeping supports_async() False.
"""
return self.supports_async()
def supports_deferred_moe_finalize(self) -> bool:
"""Whether ``finalize`` can be skipped for a deferring consumer.
@@ -1206,7 +1215,7 @@ class FusedMoEKernelModularImpl:
# We shouldn't be running an a2a kernel that doesn't
# support async prepare/finalize
# TODO(lucas): enable in follow-up
assert not dbo_enabled()
assert not dbo_enabled() or self.prepare_finalize.supports_dbo()
(
a1q,
@@ -1394,7 +1403,7 @@ class FusedMoEKernelModularImpl:
"""
if not self.prepare_finalize.supports_async():
assert not dbo_enabled()
assert not dbo_enabled() or self.prepare_finalize.supports_dbo()
self.prepare_finalize.finalize(
output,
@@ -1,5 +1,8 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Iterator
from contextlib import contextmanager
import torch
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
@@ -11,7 +14,13 @@ from vllm.model_executor.layers.fused_moe.topk_weight_and_reduce import (
TopKWeightAndReduceNoOP,
)
from vllm.model_executor.layers.fused_moe.utils import moe_kernel_quantize_input
from vllm.platforms import current_platform
from vllm.utils.flashinfer import nvfp4_block_scale_interleave
from vllm.v1.worker.ubatching import (
dbo_enabled,
dbo_switch_to_compute_sync,
dbo_yield_and_switch_from_compute_to_comm,
)
def _quantize_and_setup_dispatch(
@@ -133,6 +142,34 @@ class MoEPrepareAndFinalizeNaiveDPEPModular(mk.FusedMoEPrepareAndFinalizeModular
return self._allocate_combine_input(shape, dtype, device)
return None
def supports_dbo(self) -> bool:
# prepare()/finalize() hand the collective to the comm stream
# themselves (see _comm_region), so the synchronous path is DBO-safe.
# supports_async() stays False: it also selects MK-internal
# shared-expert overlap. Only validated on ROCm so far.
return current_platform.is_rocm()
@contextmanager
def _comm_region(self) -> Iterator[None]:
"""Scope the dispatch/combine collective runs in.
Without DBO the collective runs on the calling stream. Under DBO it
runs on the comm stream so the peer ubatch can compute while it is in
flight, using the same compute/comm handoff as DeepEP-HT. Graph
capture keeps every collective on the capture stream, so overlap is
limited to eager steps (prefill).
"""
if not dbo_enabled() or torch.cuda.is_current_stream_capturing():
yield
return
# Release the CPU to the peer ubatch so it can enqueue compute while
# this ubatch's collective runs; the compute stream waits on exit.
dbo_yield_and_switch_from_compute_to_comm()
try:
yield
finally:
dbo_switch_to_compute_sync()
def prepare(
self,
a1: torch.Tensor,
@@ -178,13 +215,14 @@ class MoEPrepareAndFinalizeNaiveDPEPModular(mk.FusedMoEPrepareAndFinalizeModular
extra_tensors = []
extra_tensors.append(local_token_lora_mapping)
res = get_ep_group().dispatch(
a1q,
topk_weights,
topk_ids,
is_sequence_parallel=self.is_sequence_parallel,
extra_tensors=extra_tensors,
)
with self._comm_region():
res = get_ep_group().dispatch(
a1q,
topk_weights,
topk_ids,
is_sequence_parallel=self.is_sequence_parallel,
extra_tensors=extra_tensors,
)
if extra_tensors is None:
assert len(res) == 3
@@ -238,11 +276,12 @@ class MoEPrepareAndFinalizeNaiveDPEPModular(mk.FusedMoEPrepareAndFinalizeModular
apply_router_weight_on_input=apply_router_weight_on_input,
)
get_ep_group().combine_into_output(
out,
output,
is_sequence_parallel=self.is_sequence_parallel,
)
with self._comm_region():
get_ep_group().combine_into_output(
out,
output,
is_sequence_parallel=self.is_sequence_parallel,
)
class MoEPrepareAndFinalizeNaiveDPEPMonolithic(mk.FusedMoEPrepareAndFinalizeMonolithic):