[Kernel][MoE] DeepEP v2: async finalize to overlap shared experts with combine (#52781)

Signed-off-by: Mario <404mario@users.noreply.github.com>
Co-authored-by: Mario <404mario@users.noreply.github.com>
Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
404mario
2026-09-15 00:04:22 -04:00
committed by GitHub
co-authored by Mario Claude Fable 5
parent a72c046463
commit 9d14fd3095
2 changed files with 150 additions and 8 deletions
@@ -21,6 +21,7 @@ from vllm.triton_utils import tl, triton
from vllm.utils.math_utils import round_up
from vllm.v1.worker.ubatching import (
dbo_current_ubatch_id,
dbo_enabled,
)
@@ -89,8 +90,10 @@ class DeepEPV2PrepareAndFinalize(mk.FusedMoEPrepareAndFinalizeModular):
use cudagraphs anyway
- Provides expert_tokens_meta for efficient batched expert kernels
Both modes use async_with_compute_stream=False (synchronous from
caller's perspective). The ElasticBuffer handles comm internally.
Dispatch always uses async_with_compute_stream=False. finalize_async
issues the combine with async_with_compute_stream=True (except under
DBO) so the modular kernel can overlap the shared-expert FFN with the
combine a2a; the returned receiver joins via a device-side event wait.
"""
@staticmethod
@@ -465,15 +468,31 @@ class DeepEPV2PrepareAndFinalize(mk.FusedMoEPrepareAndFinalizeModular):
f"got {fused_expert_output.dtype}"
)
# DBO drives its own hook/receiver schedule; keep the combine
# synchronous there (the receiver then only performs the copy).
combine_async = do_async and not dbo_enabled()
combined_x, _, event = self.buffer.combine(
x=fused_expert_output,
handle=handle,
topk_weights=None,
async_with_compute_stream=False,
async_with_compute_stream=combine_async,
allocate_on_comm_stream=combine_async,
)
output.copy_(combined_x, non_blocking=True)
return None
if do_async:
# The combine ran on DeepEP's comm stream; the modular kernel
# issues the shared-expert FFN before calling the receiver, which
# joins via a device-side cudaStreamWaitEvent (no host sync, so
# this is safe inside a captured region).
def _receiver():
if event.event is not None:
event.current_stream_wait()
output.copy_(combined_x, non_blocking=True)
return _receiver
else:
output.copy_(combined_x, non_blocking=True)
return None
def finalize_async(
self,
@@ -484,16 +503,17 @@ class DeepEPV2PrepareAndFinalize(mk.FusedMoEPrepareAndFinalizeModular):
apply_router_weight_on_input: bool,
weight_and_reduce_impl: mk.TopKWeightAndReduce,
) -> Callable:
self._finalize(
receiver = self._finalize(
output,
fused_expert_output,
topk_weights,
topk_ids,
apply_router_weight_on_input,
weight_and_reduce_impl,
False,
True,
)
return lambda: None
assert receiver is not None
return receiver
def finalize(
self,