[ROCm][DeepEP] Stabilize high-throughput DBO for DP+EP (#46990)
Signed-off-by: Andreas Karatzas <akaratza@amd.com> Signed-off-by: Andreas Karatzas <Andreas.Karatzas@amd.com> Co-authored-by: Tyler Michael Smith <tlrmchlsmth@gmail.com>
This commit is contained in:
co-authored by
Tyler Michael Smith
parent
4eb227992a
commit
53f7553f09
@@ -12,6 +12,7 @@ from vllm.model_executor.layers.fused_moe.topk_weight_and_reduce import (
|
||||
TopKWeightAndReduceDelegate,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.utils import moe_kernel_quantize_input
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.math_utils import round_up
|
||||
from vllm.v1.worker.ubatching import (
|
||||
dbo_current_ubatch_id,
|
||||
@@ -59,6 +60,7 @@ class DeepEPHTPrepareAndFinalize(mk.FusedMoEPrepareAndFinalizeModular):
|
||||
self.dp_size = dp_size
|
||||
self.rank_expert_offset = rank_expert_offset
|
||||
self.async_prepare = True
|
||||
self.sync_dbo_comm = current_platform.is_rocm()
|
||||
|
||||
# The dispatch function returns a handle that the combine function
|
||||
# requires. Under DBO microbatching we must track one handle per
|
||||
@@ -68,6 +70,13 @@ class DeepEPHTPrepareAndFinalize(mk.FusedMoEPrepareAndFinalizeModular):
|
||||
# From https://github.com/deepseek-ai/DeepEP/blob/9fe9021f29c9083cd1808ab36b740208524d9f63/deep_ep/buffer.py#L164
|
||||
self.available_rank_configs = [2, 4, 8, 16, 24, 32, 64, 128, 144, 160]
|
||||
|
||||
def _sync_dbo_comm_if_needed(self) -> None:
|
||||
if self.sync_dbo_comm and dbo_enabled():
|
||||
# ROCm DeepEP HT dispatch/combine reuse Buffer-owned communication
|
||||
# workspace. Do not let the next DBO ubatch reuse that workspace
|
||||
# before this ubatch's HT kernel has completed.
|
||||
torch.cuda.current_stream().synchronize()
|
||||
|
||||
def num_dispatchers(self) -> int:
|
||||
return self.num_dispatchers_
|
||||
|
||||
@@ -161,6 +170,8 @@ class DeepEPHTPrepareAndFinalize(mk.FusedMoEPrepareAndFinalizeModular):
|
||||
allocate_on_comm_stream=False,
|
||||
)
|
||||
|
||||
self._sync_dbo_comm_if_needed()
|
||||
|
||||
# record the handle for this ubatch
|
||||
a2a_idx = dbo_current_ubatch_id()
|
||||
self.handles[a2a_idx] = handle
|
||||
@@ -375,6 +386,8 @@ class DeepEPHTPrepareAndFinalize(mk.FusedMoEPrepareAndFinalizeModular):
|
||||
allocate_on_comm_stream=False,
|
||||
)
|
||||
|
||||
self._sync_dbo_comm_if_needed()
|
||||
|
||||
dbo_switch_to_compute()
|
||||
|
||||
if do_async:
|
||||
|
||||
Reference in New Issue
Block a user