[ROCm][Perf] Enable DPA+ETP prefill compute/comm dual-batch-overlap for DeepSeek-V4 (#57773)

Signed-off-by: Shanshan Shen <87969357+shen-shanshan@users.noreply.github.com>
Signed-off-by: shen-shanshan <467638484@qq.com>
This commit is contained in:
Shanshan Shen
2026-10-15 11:16:48 +08:00
committed by GitHub
parent 5d5c3d7c43
commit f46021a7fa
9 changed files with 319 additions and 28 deletions
+22 -11
View File
@@ -2116,17 +2116,28 @@ class VllmConfig:
if self.parallel_config.use_ubatching:
a2a_backend = self.parallel_config.all2all_backend
assert a2a_backend in [
"deepep_low_latency",
"deepep_high_throughput",
"nixl_ep",
], (
"Microbatching currently only supports the deepep_low_latency, "
"deepep_high_throughput, and nixl_ep all2all backends. "
f"{a2a_backend} is not supported. To fix use "
"--all2all-backend=deepep_low_latency, "
"--all2all-backend=deepep_high_throughput, or "
"--all2all-backend=nixl_ep and install the matching kernels."
if current_platform.is_rocm():
supported_backends = [
"deepep_low_latency",
"deepep_high_throughput",
"nixl_ep",
# The naive DP/EP prepare/finalize runs its all-gather /
# reduce-scatter on the comm stream under DBO.
"allgather_reducescatter",
]
else:
# CUDA and other platforms.
supported_backends = [
"deepep_low_latency",
"deepep_high_throughput",
"nixl_ep",
]
assert a2a_backend in supported_backends, (
"Microbatching currently only supports the "
f"{', '.join(supported_backends)} all2all backends on this "
f"platform. {a2a_backend} is not supported. To fix use one of "
f"{', '.join('--all2all-backend=' + b for b in supported_backends)}"
" and install the matching kernels."
)
if not self.instance_id:
@@ -258,6 +258,15 @@ class FusedMoEPrepareAndFinalize(ABC):
"""
return False
def supports_dbo(self) -> bool:
"""Indicates whether or not this class can run under DBO.
Defaults to supports_async(). A class whose synchronous prepare and
finalize do their own DBO compute/comm handoff can return True here
while keeping supports_async() False.
"""
return self.supports_async()
def supports_deferred_moe_finalize(self) -> bool:
"""Whether ``finalize`` can be skipped for a deferring consumer.
@@ -1206,7 +1215,7 @@ class FusedMoEKernelModularImpl:
# We shouldn't be running an a2a kernel that doesn't
# support async prepare/finalize
# TODO(lucas): enable in follow-up
assert not dbo_enabled()
assert not dbo_enabled() or self.prepare_finalize.supports_dbo()
(
a1q,
@@ -1394,7 +1403,7 @@ class FusedMoEKernelModularImpl:
"""
if not self.prepare_finalize.supports_async():
assert not dbo_enabled()
assert not dbo_enabled() or self.prepare_finalize.supports_dbo()
self.prepare_finalize.finalize(
output,
@@ -1,5 +1,8 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Iterator
from contextlib import contextmanager
import torch
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
@@ -11,7 +14,13 @@ from vllm.model_executor.layers.fused_moe.topk_weight_and_reduce import (
TopKWeightAndReduceNoOP,
)
from vllm.model_executor.layers.fused_moe.utils import moe_kernel_quantize_input
from vllm.platforms import current_platform
from vllm.utils.flashinfer import nvfp4_block_scale_interleave
from vllm.v1.worker.ubatching import (
dbo_enabled,
dbo_switch_to_compute_sync,
dbo_yield_and_switch_from_compute_to_comm,
)
def _quantize_and_setup_dispatch(
@@ -133,6 +142,34 @@ class MoEPrepareAndFinalizeNaiveDPEPModular(mk.FusedMoEPrepareAndFinalizeModular
return self._allocate_combine_input(shape, dtype, device)
return None
def supports_dbo(self) -> bool:
# prepare()/finalize() hand the collective to the comm stream
# themselves (see _comm_region), so the synchronous path is DBO-safe.
# supports_async() stays False: it also selects MK-internal
# shared-expert overlap. Only validated on ROCm so far.
return current_platform.is_rocm()
@contextmanager
def _comm_region(self) -> Iterator[None]:
"""Scope the dispatch/combine collective runs in.
Without DBO the collective runs on the calling stream. Under DBO it
runs on the comm stream so the peer ubatch can compute while it is in
flight, using the same compute/comm handoff as DeepEP-HT. Graph
capture keeps every collective on the capture stream, so overlap is
limited to eager steps (prefill).
"""
if not dbo_enabled() or torch.cuda.is_current_stream_capturing():
yield
return
# Release the CPU to the peer ubatch so it can enqueue compute while
# this ubatch's collective runs; the compute stream waits on exit.
dbo_yield_and_switch_from_compute_to_comm()
try:
yield
finally:
dbo_switch_to_compute_sync()
def prepare(
self,
a1: torch.Tensor,
@@ -178,13 +215,14 @@ class MoEPrepareAndFinalizeNaiveDPEPModular(mk.FusedMoEPrepareAndFinalizeModular
extra_tensors = []
extra_tensors.append(local_token_lora_mapping)
res = get_ep_group().dispatch(
a1q,
topk_weights,
topk_ids,
is_sequence_parallel=self.is_sequence_parallel,
extra_tensors=extra_tensors,
)
with self._comm_region():
res = get_ep_group().dispatch(
a1q,
topk_weights,
topk_ids,
is_sequence_parallel=self.is_sequence_parallel,
extra_tensors=extra_tensors,
)
if extra_tensors is None:
assert len(res) == 3
@@ -238,11 +276,12 @@ class MoEPrepareAndFinalizeNaiveDPEPModular(mk.FusedMoEPrepareAndFinalizeModular
apply_router_weight_on_input=apply_router_weight_on_input,
)
get_ep_group().combine_into_output(
out,
output,
is_sequence_parallel=self.is_sequence_parallel,
)
with self._comm_region():
get_ep_group().combine_into_output(
out,
output,
is_sequence_parallel=self.is_sequence_parallel,
)
class MoEPrepareAndFinalizeNaiveDPEPMonolithic(mk.FusedMoEPrepareAndFinalizeMonolithic):
+8 -1
View File
@@ -82,6 +82,7 @@ from vllm.models.deepseek_v4.amd.rocm import DeepseekV4ROCMAiterMLAAttention
from vllm.platforms import current_platform
from vllm.platforms.rocm import on_gfx950
from vllm.sequence import IntermediateTensors
from vllm.v1.worker.ubatching import dbo_current_ubatch_token_offset
from ..common.mm_preprocess import IMAGE_SENTINEL_BASE_ID
@@ -1228,7 +1229,13 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
if self._mtp_hidden_buffer is not None:
num_tokens = hidden_states.shape[0]
self._mtp_hidden_buffer[:num_tokens].copy_(hidden_states.flatten(1))
# Each DBO ubatch owns a different slice of the batch. Writing
# from row 0 in both threads would let one ubatch clobber the
# other's rows in the buffer the MTP drafter reads after the step.
offset = dbo_current_ubatch_token_offset()
self._mtp_hidden_buffer[offset : offset + num_tokens].copy_(
hidden_states.flatten(1)
)
hidden_states = self.hc_head_op(
hidden_states,
+2
View File
@@ -213,6 +213,7 @@ class UBatchWrapper:
def _run_ubatches(self, ubatch_metadata, model) -> torch.Tensor:
@torch.inference_mode()
def _ubatch_thread(results, model, ubatch_metadata):
torch.accelerator.set_device_index(self.device)
with ubatch_metadata.context:
model_output = model(
input_ids=ubatch_metadata.input_ids,
@@ -291,6 +292,7 @@ class UBatchWrapper:
compute_stream=compute_stream,
forward_contexts=forward_contexts,
ready_barrier=self.ready_barrier,
token_offsets=[s.token_slice.start for s in ubatch_slices],
)
ubatch_metadata: list[UbatchMetadata] = []
+12 -2
View File
@@ -176,7 +176,12 @@ def maybe_create_ubatch_slices(
if split_point is None:
split_point = int(num_tokens_padded) // num_ubatches
token_split_points = [split_point * i for i in range(1, num_ubatches)]
if isinstance(split_point, int):
token_split_points = [split_point * i for i in range(1, num_ubatches)]
else:
# Explicit split points, one per ubatch boundary.
assert len(split_point) == num_ubatches - 1
token_split_points = [int(p) for p in split_point]
# TODO(lucas): Refactor the gpu_model_runner.py so we can pass
# in cu_num_tokens directly (i.e. query_start_loc)
@@ -187,7 +192,7 @@ def maybe_create_ubatch_slices(
start_token = 0
# Add the end point to the split points to make iteration easier
all_points = token_split_points + [cu_num_tokens[-1]]
all_points = token_split_points + [int(cu_num_tokens[-1])]
for end_token in all_points:
token_slice = slice(start_token, end_token)
@@ -316,6 +321,10 @@ def _make_metadata_with_slice(
block_table_tensor = attn_metadata.block_table_tensor[request_slice]
slot_mapping = attn_metadata.slot_mapping[token_slice]
positions = attn_metadata.positions
if positions is not None:
positions = positions[token_slice]
return CommonAttentionMetadata(
query_start_loc=query_start_loc,
query_start_loc_cpu=query_start_loc_cpu,
@@ -326,6 +335,7 @@ def _make_metadata_with_slice(
max_seq_len=max_seq_len,
block_table_tensor=block_table_tensor,
slot_mapping=slot_mapping,
positions=positions,
seq_lens_cpu_upper_bound=seq_lens_cpu_upper_bound,
)
+15
View File
@@ -32,6 +32,7 @@ class UBatchContext:
gpu_comm_done_event: torch.Event,
gpu_compute_done_event: torch.Event,
schedule: str = "default",
token_offset: int = 0,
):
self.id = id
self.comm_stream = comm_stream
@@ -44,6 +45,7 @@ class UBatchContext:
self.gpu_comm_done_event = gpu_comm_done_event
self.gpu_compute_done_event = gpu_compute_done_event
self.schedule = schedule
self.token_offset = token_offset
self.recv_hook = None
def __enter__(self):
@@ -155,6 +157,17 @@ def dbo_current_ubatch_id() -> int:
return _THREAD_ID_TO_CONTEXT[threading.get_ident()]
def dbo_current_ubatch_token_offset() -> int:
"""Row offset of the calling ubatch inside the parent batch.
Returns 0 when no DBO context is active on this thread.
"""
if len(_THREAD_ID_TO_CONTEXT) == 0:
return 0
ctx = _CURRENT_CONTEXTS[_THREAD_ID_TO_CONTEXT[threading.get_ident()]]
return ctx.token_offset
def _register_ubatch_function(func):
def wrapper(*args, **kwargs):
if len(_THREAD_ID_TO_CONTEXT) > 0:
@@ -204,6 +217,7 @@ def make_ubatch_contexts(
forward_contexts: list[ForwardContext],
ready_barrier: threading.Barrier,
schedule: str = "default",
token_offsets: list[int] | None = None,
) -> list[UBatchContext]:
global _NUM_UBATCHES, _CURRENT_CONTEXTS
assert num_micro_batches > 1, "num_micro_batches must be greater than 1"
@@ -233,6 +247,7 @@ def make_ubatch_contexts(
gpu_comm_done_event=gpu_comm_done_events[i],
gpu_compute_done_event=gpu_compute_done_events[i],
schedule=schedule,
token_offset=token_offsets[i] if token_offsets else 0,
)
ctxs.append(ctx)