[ROCm][Perf] Enable DPA+ETP prefill compute/comm dual-batch-overlap for DeepSeek-V4 (#57773)
Signed-off-by: Shanshan Shen <87969357+shen-shanshan@users.noreply.github.com> Signed-off-by: shen-shanshan <467638484@qq.com>
This commit is contained in:
@@ -258,6 +258,15 @@ class FusedMoEPrepareAndFinalize(ABC):
|
||||
"""
|
||||
return False
|
||||
|
||||
def supports_dbo(self) -> bool:
|
||||
"""Indicates whether or not this class can run under DBO.
|
||||
|
||||
Defaults to supports_async(). A class whose synchronous prepare and
|
||||
finalize do their own DBO compute/comm handoff can return True here
|
||||
while keeping supports_async() False.
|
||||
"""
|
||||
return self.supports_async()
|
||||
|
||||
def supports_deferred_moe_finalize(self) -> bool:
|
||||
"""Whether ``finalize`` can be skipped for a deferring consumer.
|
||||
|
||||
@@ -1206,7 +1215,7 @@ class FusedMoEKernelModularImpl:
|
||||
# We shouldn't be running an a2a kernel that doesn't
|
||||
# support async prepare/finalize
|
||||
# TODO(lucas): enable in follow-up
|
||||
assert not dbo_enabled()
|
||||
assert not dbo_enabled() or self.prepare_finalize.supports_dbo()
|
||||
|
||||
(
|
||||
a1q,
|
||||
@@ -1394,7 +1403,7 @@ class FusedMoEKernelModularImpl:
|
||||
|
||||
"""
|
||||
if not self.prepare_finalize.supports_async():
|
||||
assert not dbo_enabled()
|
||||
assert not dbo_enabled() or self.prepare_finalize.supports_dbo()
|
||||
|
||||
self.prepare_finalize.finalize(
|
||||
output,
|
||||
|
||||
@@ -1,5 +1,8 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from collections.abc import Iterator
|
||||
from contextlib import contextmanager
|
||||
|
||||
import torch
|
||||
|
||||
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
|
||||
@@ -11,7 +14,13 @@ from vllm.model_executor.layers.fused_moe.topk_weight_and_reduce import (
|
||||
TopKWeightAndReduceNoOP,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.utils import moe_kernel_quantize_input
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.flashinfer import nvfp4_block_scale_interleave
|
||||
from vllm.v1.worker.ubatching import (
|
||||
dbo_enabled,
|
||||
dbo_switch_to_compute_sync,
|
||||
dbo_yield_and_switch_from_compute_to_comm,
|
||||
)
|
||||
|
||||
|
||||
def _quantize_and_setup_dispatch(
|
||||
@@ -133,6 +142,34 @@ class MoEPrepareAndFinalizeNaiveDPEPModular(mk.FusedMoEPrepareAndFinalizeModular
|
||||
return self._allocate_combine_input(shape, dtype, device)
|
||||
return None
|
||||
|
||||
def supports_dbo(self) -> bool:
|
||||
# prepare()/finalize() hand the collective to the comm stream
|
||||
# themselves (see _comm_region), so the synchronous path is DBO-safe.
|
||||
# supports_async() stays False: it also selects MK-internal
|
||||
# shared-expert overlap. Only validated on ROCm so far.
|
||||
return current_platform.is_rocm()
|
||||
|
||||
@contextmanager
|
||||
def _comm_region(self) -> Iterator[None]:
|
||||
"""Scope the dispatch/combine collective runs in.
|
||||
|
||||
Without DBO the collective runs on the calling stream. Under DBO it
|
||||
runs on the comm stream so the peer ubatch can compute while it is in
|
||||
flight, using the same compute/comm handoff as DeepEP-HT. Graph
|
||||
capture keeps every collective on the capture stream, so overlap is
|
||||
limited to eager steps (prefill).
|
||||
"""
|
||||
if not dbo_enabled() or torch.cuda.is_current_stream_capturing():
|
||||
yield
|
||||
return
|
||||
# Release the CPU to the peer ubatch so it can enqueue compute while
|
||||
# this ubatch's collective runs; the compute stream waits on exit.
|
||||
dbo_yield_and_switch_from_compute_to_comm()
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
dbo_switch_to_compute_sync()
|
||||
|
||||
def prepare(
|
||||
self,
|
||||
a1: torch.Tensor,
|
||||
@@ -178,13 +215,14 @@ class MoEPrepareAndFinalizeNaiveDPEPModular(mk.FusedMoEPrepareAndFinalizeModular
|
||||
extra_tensors = []
|
||||
extra_tensors.append(local_token_lora_mapping)
|
||||
|
||||
res = get_ep_group().dispatch(
|
||||
a1q,
|
||||
topk_weights,
|
||||
topk_ids,
|
||||
is_sequence_parallel=self.is_sequence_parallel,
|
||||
extra_tensors=extra_tensors,
|
||||
)
|
||||
with self._comm_region():
|
||||
res = get_ep_group().dispatch(
|
||||
a1q,
|
||||
topk_weights,
|
||||
topk_ids,
|
||||
is_sequence_parallel=self.is_sequence_parallel,
|
||||
extra_tensors=extra_tensors,
|
||||
)
|
||||
|
||||
if extra_tensors is None:
|
||||
assert len(res) == 3
|
||||
@@ -238,11 +276,12 @@ class MoEPrepareAndFinalizeNaiveDPEPModular(mk.FusedMoEPrepareAndFinalizeModular
|
||||
apply_router_weight_on_input=apply_router_weight_on_input,
|
||||
)
|
||||
|
||||
get_ep_group().combine_into_output(
|
||||
out,
|
||||
output,
|
||||
is_sequence_parallel=self.is_sequence_parallel,
|
||||
)
|
||||
with self._comm_region():
|
||||
get_ep_group().combine_into_output(
|
||||
out,
|
||||
output,
|
||||
is_sequence_parallel=self.is_sequence_parallel,
|
||||
)
|
||||
|
||||
|
||||
class MoEPrepareAndFinalizeNaiveDPEPMonolithic(mk.FusedMoEPrepareAndFinalizeMonolithic):
|
||||
|
||||
Reference in New Issue
Block a user