[ROCm][Perf] Enable DPA+ETP prefill compute/comm dual-batch-overlap for DeepSeek-V4 (#57773)
Signed-off-by: Shanshan Shen <87969357+shen-shanshan@users.noreply.github.com> Signed-off-by: shen-shanshan <467638484@qq.com>
This commit is contained in:
+22
-11
@@ -2116,17 +2116,28 @@ class VllmConfig:
|
||||
|
||||
if self.parallel_config.use_ubatching:
|
||||
a2a_backend = self.parallel_config.all2all_backend
|
||||
assert a2a_backend in [
|
||||
"deepep_low_latency",
|
||||
"deepep_high_throughput",
|
||||
"nixl_ep",
|
||||
], (
|
||||
"Microbatching currently only supports the deepep_low_latency, "
|
||||
"deepep_high_throughput, and nixl_ep all2all backends. "
|
||||
f"{a2a_backend} is not supported. To fix use "
|
||||
"--all2all-backend=deepep_low_latency, "
|
||||
"--all2all-backend=deepep_high_throughput, or "
|
||||
"--all2all-backend=nixl_ep and install the matching kernels."
|
||||
if current_platform.is_rocm():
|
||||
supported_backends = [
|
||||
"deepep_low_latency",
|
||||
"deepep_high_throughput",
|
||||
"nixl_ep",
|
||||
# The naive DP/EP prepare/finalize runs its all-gather /
|
||||
# reduce-scatter on the comm stream under DBO.
|
||||
"allgather_reducescatter",
|
||||
]
|
||||
else:
|
||||
# CUDA and other platforms.
|
||||
supported_backends = [
|
||||
"deepep_low_latency",
|
||||
"deepep_high_throughput",
|
||||
"nixl_ep",
|
||||
]
|
||||
assert a2a_backend in supported_backends, (
|
||||
"Microbatching currently only supports the "
|
||||
f"{', '.join(supported_backends)} all2all backends on this "
|
||||
f"platform. {a2a_backend} is not supported. To fix use one of "
|
||||
f"{', '.join('--all2all-backend=' + b for b in supported_backends)}"
|
||||
" and install the matching kernels."
|
||||
)
|
||||
|
||||
if not self.instance_id:
|
||||
|
||||
@@ -258,6 +258,15 @@ class FusedMoEPrepareAndFinalize(ABC):
|
||||
"""
|
||||
return False
|
||||
|
||||
def supports_dbo(self) -> bool:
|
||||
"""Indicates whether or not this class can run under DBO.
|
||||
|
||||
Defaults to supports_async(). A class whose synchronous prepare and
|
||||
finalize do their own DBO compute/comm handoff can return True here
|
||||
while keeping supports_async() False.
|
||||
"""
|
||||
return self.supports_async()
|
||||
|
||||
def supports_deferred_moe_finalize(self) -> bool:
|
||||
"""Whether ``finalize`` can be skipped for a deferring consumer.
|
||||
|
||||
@@ -1206,7 +1215,7 @@ class FusedMoEKernelModularImpl:
|
||||
# We shouldn't be running an a2a kernel that doesn't
|
||||
# support async prepare/finalize
|
||||
# TODO(lucas): enable in follow-up
|
||||
assert not dbo_enabled()
|
||||
assert not dbo_enabled() or self.prepare_finalize.supports_dbo()
|
||||
|
||||
(
|
||||
a1q,
|
||||
@@ -1394,7 +1403,7 @@ class FusedMoEKernelModularImpl:
|
||||
|
||||
"""
|
||||
if not self.prepare_finalize.supports_async():
|
||||
assert not dbo_enabled()
|
||||
assert not dbo_enabled() or self.prepare_finalize.supports_dbo()
|
||||
|
||||
self.prepare_finalize.finalize(
|
||||
output,
|
||||
|
||||
@@ -1,5 +1,8 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from collections.abc import Iterator
|
||||
from contextlib import contextmanager
|
||||
|
||||
import torch
|
||||
|
||||
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
|
||||
@@ -11,7 +14,13 @@ from vllm.model_executor.layers.fused_moe.topk_weight_and_reduce import (
|
||||
TopKWeightAndReduceNoOP,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.utils import moe_kernel_quantize_input
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.flashinfer import nvfp4_block_scale_interleave
|
||||
from vllm.v1.worker.ubatching import (
|
||||
dbo_enabled,
|
||||
dbo_switch_to_compute_sync,
|
||||
dbo_yield_and_switch_from_compute_to_comm,
|
||||
)
|
||||
|
||||
|
||||
def _quantize_and_setup_dispatch(
|
||||
@@ -133,6 +142,34 @@ class MoEPrepareAndFinalizeNaiveDPEPModular(mk.FusedMoEPrepareAndFinalizeModular
|
||||
return self._allocate_combine_input(shape, dtype, device)
|
||||
return None
|
||||
|
||||
def supports_dbo(self) -> bool:
|
||||
# prepare()/finalize() hand the collective to the comm stream
|
||||
# themselves (see _comm_region), so the synchronous path is DBO-safe.
|
||||
# supports_async() stays False: it also selects MK-internal
|
||||
# shared-expert overlap. Only validated on ROCm so far.
|
||||
return current_platform.is_rocm()
|
||||
|
||||
@contextmanager
|
||||
def _comm_region(self) -> Iterator[None]:
|
||||
"""Scope the dispatch/combine collective runs in.
|
||||
|
||||
Without DBO the collective runs on the calling stream. Under DBO it
|
||||
runs on the comm stream so the peer ubatch can compute while it is in
|
||||
flight, using the same compute/comm handoff as DeepEP-HT. Graph
|
||||
capture keeps every collective on the capture stream, so overlap is
|
||||
limited to eager steps (prefill).
|
||||
"""
|
||||
if not dbo_enabled() or torch.cuda.is_current_stream_capturing():
|
||||
yield
|
||||
return
|
||||
# Release the CPU to the peer ubatch so it can enqueue compute while
|
||||
# this ubatch's collective runs; the compute stream waits on exit.
|
||||
dbo_yield_and_switch_from_compute_to_comm()
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
dbo_switch_to_compute_sync()
|
||||
|
||||
def prepare(
|
||||
self,
|
||||
a1: torch.Tensor,
|
||||
@@ -178,13 +215,14 @@ class MoEPrepareAndFinalizeNaiveDPEPModular(mk.FusedMoEPrepareAndFinalizeModular
|
||||
extra_tensors = []
|
||||
extra_tensors.append(local_token_lora_mapping)
|
||||
|
||||
res = get_ep_group().dispatch(
|
||||
a1q,
|
||||
topk_weights,
|
||||
topk_ids,
|
||||
is_sequence_parallel=self.is_sequence_parallel,
|
||||
extra_tensors=extra_tensors,
|
||||
)
|
||||
with self._comm_region():
|
||||
res = get_ep_group().dispatch(
|
||||
a1q,
|
||||
topk_weights,
|
||||
topk_ids,
|
||||
is_sequence_parallel=self.is_sequence_parallel,
|
||||
extra_tensors=extra_tensors,
|
||||
)
|
||||
|
||||
if extra_tensors is None:
|
||||
assert len(res) == 3
|
||||
@@ -238,11 +276,12 @@ class MoEPrepareAndFinalizeNaiveDPEPModular(mk.FusedMoEPrepareAndFinalizeModular
|
||||
apply_router_weight_on_input=apply_router_weight_on_input,
|
||||
)
|
||||
|
||||
get_ep_group().combine_into_output(
|
||||
out,
|
||||
output,
|
||||
is_sequence_parallel=self.is_sequence_parallel,
|
||||
)
|
||||
with self._comm_region():
|
||||
get_ep_group().combine_into_output(
|
||||
out,
|
||||
output,
|
||||
is_sequence_parallel=self.is_sequence_parallel,
|
||||
)
|
||||
|
||||
|
||||
class MoEPrepareAndFinalizeNaiveDPEPMonolithic(mk.FusedMoEPrepareAndFinalizeMonolithic):
|
||||
|
||||
@@ -82,6 +82,7 @@ from vllm.models.deepseek_v4.amd.rocm import DeepseekV4ROCMAiterMLAAttention
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.platforms.rocm import on_gfx950
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.v1.worker.ubatching import dbo_current_ubatch_token_offset
|
||||
|
||||
from ..common.mm_preprocess import IMAGE_SENTINEL_BASE_ID
|
||||
|
||||
@@ -1228,7 +1229,13 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
|
||||
|
||||
if self._mtp_hidden_buffer is not None:
|
||||
num_tokens = hidden_states.shape[0]
|
||||
self._mtp_hidden_buffer[:num_tokens].copy_(hidden_states.flatten(1))
|
||||
# Each DBO ubatch owns a different slice of the batch. Writing
|
||||
# from row 0 in both threads would let one ubatch clobber the
|
||||
# other's rows in the buffer the MTP drafter reads after the step.
|
||||
offset = dbo_current_ubatch_token_offset()
|
||||
self._mtp_hidden_buffer[offset : offset + num_tokens].copy_(
|
||||
hidden_states.flatten(1)
|
||||
)
|
||||
|
||||
hidden_states = self.hc_head_op(
|
||||
hidden_states,
|
||||
|
||||
@@ -213,6 +213,7 @@ class UBatchWrapper:
|
||||
def _run_ubatches(self, ubatch_metadata, model) -> torch.Tensor:
|
||||
@torch.inference_mode()
|
||||
def _ubatch_thread(results, model, ubatch_metadata):
|
||||
torch.accelerator.set_device_index(self.device)
|
||||
with ubatch_metadata.context:
|
||||
model_output = model(
|
||||
input_ids=ubatch_metadata.input_ids,
|
||||
@@ -291,6 +292,7 @@ class UBatchWrapper:
|
||||
compute_stream=compute_stream,
|
||||
forward_contexts=forward_contexts,
|
||||
ready_barrier=self.ready_barrier,
|
||||
token_offsets=[s.token_slice.start for s in ubatch_slices],
|
||||
)
|
||||
|
||||
ubatch_metadata: list[UbatchMetadata] = []
|
||||
|
||||
@@ -176,7 +176,12 @@ def maybe_create_ubatch_slices(
|
||||
if split_point is None:
|
||||
split_point = int(num_tokens_padded) // num_ubatches
|
||||
|
||||
token_split_points = [split_point * i for i in range(1, num_ubatches)]
|
||||
if isinstance(split_point, int):
|
||||
token_split_points = [split_point * i for i in range(1, num_ubatches)]
|
||||
else:
|
||||
# Explicit split points, one per ubatch boundary.
|
||||
assert len(split_point) == num_ubatches - 1
|
||||
token_split_points = [int(p) for p in split_point]
|
||||
|
||||
# TODO(lucas): Refactor the gpu_model_runner.py so we can pass
|
||||
# in cu_num_tokens directly (i.e. query_start_loc)
|
||||
@@ -187,7 +192,7 @@ def maybe_create_ubatch_slices(
|
||||
start_token = 0
|
||||
|
||||
# Add the end point to the split points to make iteration easier
|
||||
all_points = token_split_points + [cu_num_tokens[-1]]
|
||||
all_points = token_split_points + [int(cu_num_tokens[-1])]
|
||||
|
||||
for end_token in all_points:
|
||||
token_slice = slice(start_token, end_token)
|
||||
@@ -316,6 +321,10 @@ def _make_metadata_with_slice(
|
||||
block_table_tensor = attn_metadata.block_table_tensor[request_slice]
|
||||
slot_mapping = attn_metadata.slot_mapping[token_slice]
|
||||
|
||||
positions = attn_metadata.positions
|
||||
if positions is not None:
|
||||
positions = positions[token_slice]
|
||||
|
||||
return CommonAttentionMetadata(
|
||||
query_start_loc=query_start_loc,
|
||||
query_start_loc_cpu=query_start_loc_cpu,
|
||||
@@ -326,6 +335,7 @@ def _make_metadata_with_slice(
|
||||
max_seq_len=max_seq_len,
|
||||
block_table_tensor=block_table_tensor,
|
||||
slot_mapping=slot_mapping,
|
||||
positions=positions,
|
||||
seq_lens_cpu_upper_bound=seq_lens_cpu_upper_bound,
|
||||
)
|
||||
|
||||
|
||||
@@ -32,6 +32,7 @@ class UBatchContext:
|
||||
gpu_comm_done_event: torch.Event,
|
||||
gpu_compute_done_event: torch.Event,
|
||||
schedule: str = "default",
|
||||
token_offset: int = 0,
|
||||
):
|
||||
self.id = id
|
||||
self.comm_stream = comm_stream
|
||||
@@ -44,6 +45,7 @@ class UBatchContext:
|
||||
self.gpu_comm_done_event = gpu_comm_done_event
|
||||
self.gpu_compute_done_event = gpu_compute_done_event
|
||||
self.schedule = schedule
|
||||
self.token_offset = token_offset
|
||||
self.recv_hook = None
|
||||
|
||||
def __enter__(self):
|
||||
@@ -155,6 +157,17 @@ def dbo_current_ubatch_id() -> int:
|
||||
return _THREAD_ID_TO_CONTEXT[threading.get_ident()]
|
||||
|
||||
|
||||
def dbo_current_ubatch_token_offset() -> int:
|
||||
"""Row offset of the calling ubatch inside the parent batch.
|
||||
|
||||
Returns 0 when no DBO context is active on this thread.
|
||||
"""
|
||||
if len(_THREAD_ID_TO_CONTEXT) == 0:
|
||||
return 0
|
||||
ctx = _CURRENT_CONTEXTS[_THREAD_ID_TO_CONTEXT[threading.get_ident()]]
|
||||
return ctx.token_offset
|
||||
|
||||
|
||||
def _register_ubatch_function(func):
|
||||
def wrapper(*args, **kwargs):
|
||||
if len(_THREAD_ID_TO_CONTEXT) > 0:
|
||||
@@ -204,6 +217,7 @@ def make_ubatch_contexts(
|
||||
forward_contexts: list[ForwardContext],
|
||||
ready_barrier: threading.Barrier,
|
||||
schedule: str = "default",
|
||||
token_offsets: list[int] | None = None,
|
||||
) -> list[UBatchContext]:
|
||||
global _NUM_UBATCHES, _CURRENT_CONTEXTS
|
||||
assert num_micro_batches > 1, "num_micro_batches must be greater than 1"
|
||||
@@ -233,6 +247,7 @@ def make_ubatch_contexts(
|
||||
gpu_comm_done_event=gpu_comm_done_events[i],
|
||||
gpu_compute_done_event=gpu_compute_done_events[i],
|
||||
schedule=schedule,
|
||||
token_offset=token_offsets[i] if token_offsets else 0,
|
||||
)
|
||||
ctxs.append(ctx)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user