[Kernel][MoE] DeepEP v2: async finalize to overlap shared experts with combine (#52781)
Signed-off-by: Mario <404mario@users.noreply.github.com> Co-authored-by: Mario <404mario@users.noreply.github.com> Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Mario
Claude Fable 5
parent
a72c046463
commit
9d14fd3095
@@ -21,6 +21,7 @@ from vllm.triton_utils import tl, triton
|
||||
from vllm.utils.math_utils import round_up
|
||||
from vllm.v1.worker.ubatching import (
|
||||
dbo_current_ubatch_id,
|
||||
dbo_enabled,
|
||||
)
|
||||
|
||||
|
||||
@@ -89,8 +90,10 @@ class DeepEPV2PrepareAndFinalize(mk.FusedMoEPrepareAndFinalizeModular):
|
||||
use cudagraphs anyway
|
||||
- Provides expert_tokens_meta for efficient batched expert kernels
|
||||
|
||||
Both modes use async_with_compute_stream=False (synchronous from
|
||||
caller's perspective). The ElasticBuffer handles comm internally.
|
||||
Dispatch always uses async_with_compute_stream=False. finalize_async
|
||||
issues the combine with async_with_compute_stream=True (except under
|
||||
DBO) so the modular kernel can overlap the shared-expert FFN with the
|
||||
combine a2a; the returned receiver joins via a device-side event wait.
|
||||
"""
|
||||
|
||||
@staticmethod
|
||||
@@ -465,15 +468,31 @@ class DeepEPV2PrepareAndFinalize(mk.FusedMoEPrepareAndFinalizeModular):
|
||||
f"got {fused_expert_output.dtype}"
|
||||
)
|
||||
|
||||
# DBO drives its own hook/receiver schedule; keep the combine
|
||||
# synchronous there (the receiver then only performs the copy).
|
||||
combine_async = do_async and not dbo_enabled()
|
||||
combined_x, _, event = self.buffer.combine(
|
||||
x=fused_expert_output,
|
||||
handle=handle,
|
||||
topk_weights=None,
|
||||
async_with_compute_stream=False,
|
||||
async_with_compute_stream=combine_async,
|
||||
allocate_on_comm_stream=combine_async,
|
||||
)
|
||||
|
||||
output.copy_(combined_x, non_blocking=True)
|
||||
return None
|
||||
if do_async:
|
||||
# The combine ran on DeepEP's comm stream; the modular kernel
|
||||
# issues the shared-expert FFN before calling the receiver, which
|
||||
# joins via a device-side cudaStreamWaitEvent (no host sync, so
|
||||
# this is safe inside a captured region).
|
||||
def _receiver():
|
||||
if event.event is not None:
|
||||
event.current_stream_wait()
|
||||
output.copy_(combined_x, non_blocking=True)
|
||||
|
||||
return _receiver
|
||||
else:
|
||||
output.copy_(combined_x, non_blocking=True)
|
||||
return None
|
||||
|
||||
def finalize_async(
|
||||
self,
|
||||
@@ -484,16 +503,17 @@ class DeepEPV2PrepareAndFinalize(mk.FusedMoEPrepareAndFinalizeModular):
|
||||
apply_router_weight_on_input: bool,
|
||||
weight_and_reduce_impl: mk.TopKWeightAndReduce,
|
||||
) -> Callable:
|
||||
self._finalize(
|
||||
receiver = self._finalize(
|
||||
output,
|
||||
fused_expert_output,
|
||||
topk_weights,
|
||||
topk_ids,
|
||||
apply_router_weight_on_input,
|
||||
weight_and_reduce_impl,
|
||||
False,
|
||||
True,
|
||||
)
|
||||
return lambda: None
|
||||
assert receiver is not None
|
||||
return receiver
|
||||
|
||||
def finalize(
|
||||
self,
|
||||
|
||||
Reference in New Issue
Block a user