[Perf][ROCm] Dual-stream decode with hipgraphs (#48223)

Signed-off-by: simondanielsson <simon.danielsson99@hotmail.com>
This commit is contained in:
Simon Danielsson
2026-08-12 09:32:24 +08:00
committed by GitHub
parent 466855a2bf
commit 47ececb58e
2 changed files with 54 additions and 54 deletions
@@ -573,12 +573,17 @@ class MoERunner(MoERunnerInterface):
router_logits: torch.Tensor,
shared_experts_input: torch.Tensor | None,
input_ids: torch.Tensor | None = None,
shared_experts_overlapping: bool = False,
) -> tuple[torch.Tensor | None, torch.Tensor]:
"""Run expert routing and the fused MoE kernel via the quant method.
Orchestrates shared expert execution (before/after), expert selection
via the router, and the actual fused MoE computation. Returns
(shared_expert_output, fused_expert_output).
`shared_experts_overlapping` should be True only if using multi-stream
overlap. Then the shared expert was already launched in a separate
stream, so the results only have to be awaited here.
"""
self._maybe_apply_shared_experts(
shared_experts_input, SharedExpertsOrder.NO_OVERLAP
@@ -608,10 +613,9 @@ class MoERunner(MoERunnerInterface):
shared_experts_input=shared_experts_input,
)
self._maybe_apply_shared_experts(
shared_experts_input,
SharedExpertsOrder.MULTI_STREAM_OVERLAPPED,
)
if shared_experts_overlapping:
assert self._shared_experts is not None
self._shared_experts.wait()
return (
self._shared_experts.output if self._shared_experts is not None else None,
@@ -633,18 +637,6 @@ class MoERunner(MoERunnerInterface):
else nullcontext()
)
def _maybe_sync_shared_experts_stream(
self,
shared_experts_input: torch.Tensor | None,
):
# If router/gate provided, then apply it here.
# (Note: This code runs only when "overlapped mode" is on to allow
# parallel execution of shared experts with the RoutedExperts via
# separate cuda stream)
if self._shared_experts is not None:
assert shared_experts_input is not None
self._shared_experts.maybe_sync_shared_experts_stream(shared_experts_input)
def _maybe_add_zero_expert_output(
self,
result: torch.Tensor,
@@ -849,8 +841,13 @@ class MoERunner(MoERunnerInterface):
# TODO(bnell): this can be removed after MK migration is complete.
self.routed_experts._ensure_moe_quant_config_init()
# Sync aux and main stream for shared expert multi-stream overlap.
self._maybe_sync_shared_experts_stream(shared_experts_input)
# If using multi-stream overlap for shared experts, we must launch it
# before routed expert dispatch.
shared_experts_overlapping = False
if self._shared_experts is not None:
shared_experts_overlapping = self._shared_experts.maybe_forward_async(
shared_experts_input
)
# If the Runner holds the gate, apply it after the stream sync,
# so it can run overlapped with the
@@ -876,6 +873,7 @@ class MoERunner(MoERunnerInterface):
router_logits=router_logits,
shared_experts_input=shared_experts_input,
input_ids=input_ids,
shared_experts_overlapping=shared_experts_overlapping,
)
return self._maybe_combine(
@@ -71,6 +71,11 @@ class SharedExperts(torch.nn.Module):
if self._stream is not None:
logger.debug_once("Enabled separate cuda stream for MoE shared_experts")
if self._stream is not None:
# One pair per DBO ubatch id.
self._input_ready_event = [torch.cuda.Event(), torch.cuda.Event()]
self._output_ready_event = [torch.cuda.Event(), torch.cuda.Event()]
# TODO(bnell): Hack for elastic_ep. Get rid of this
def _set_moe_config(self, new_moe_config: FusedMoEConfig):
self.moe_config = new_moe_config
@@ -96,6 +101,14 @@ class SharedExperts(torch.nn.Module):
and parallel_config.all2all_backend not in _EPLB_OVERLAP_SAFE_BACKENDS
) or parallel_config.use_fi_nvl_two_sided_kernels
@property
def _should_enable_stream_overlap_heuristic(self) -> bool:
# On ROCm, empirically it's shown that only DPA deployments benefit from
# multi-stream shared experts
if not current_platform.is_rocm():
return True
return self._moe_config.moe_parallel_config.dp_size > 1
def _determine_shared_experts_order(
self,
hidden_states: torch.Tensor,
@@ -107,10 +120,11 @@ class SharedExperts(torch.nn.Module):
return SharedExpertsOrder.MK_INTERNAL_OVERLAPPED
should_run_shared_in_aux_stream = (
current_platform.is_cuda()
current_platform.is_cuda_alike()
and self._stream is not None
and hidden_states.shape[0]
<= envs.VLLM_SHARED_EXPERTS_STREAM_TOKEN_THRESHOLD
and self._should_enable_stream_overlap_heuristic
)
if should_run_shared_in_aux_stream:
@@ -118,38 +132,31 @@ class SharedExperts(torch.nn.Module):
else:
return SharedExpertsOrder.NO_OVERLAP
def maybe_sync_shared_experts_stream(
self,
shared_experts_input: torch.Tensor,
):
experts_order = self._determine_shared_experts_order(shared_experts_input)
def maybe_forward_async(self, shared_experts_input: torch.Tensor) -> bool:
"""Enqueue shared experts on the aux stream without waiting for them.
if experts_order == SharedExpertsOrder.MULTI_STREAM_OVERLAPPED:
assert self._stream is not None
# Record that the clone will be used by shared_experts_stream
# to avoid gc issue from deallocation of hidden_states_clone
# For more details: https://docs.pytorch.org/docs/stable/generated/torch.Tensor.record_stream.html # noqa: E501
# NOTE: We don't need shared_output.record_stream(current_stream())
# because we synch the streams before using shared_output.
shared_experts_input.record_stream(self._stream)
# Mark sync start point for the aux stream since we will
# run in parallel with router/gate.
self._stream.wait_stream(current_stream())
def _run_in_aux_stream(
self,
shared_experts_input: torch.Tensor,
) -> torch.Tensor:
# TODO: assert that maybe_sync_shared_experts_stream has been called.
# Run shared experts in parallel on a separate stream.
Returns true if the shared experts were enqueued, false otherwise. Call
`wait` to wait for the shared experts to finish if this returns true.
"""
if (
self._determine_shared_experts_order(shared_experts_input)
!= SharedExpertsOrder.MULTI_STREAM_OVERLAPPED
):
return False
assert self._stream is not None
idx = self._output_idx
assert self._output[idx] is None
self._input_ready_event[idx].record(current_stream())
with torch.cuda.stream(self._stream):
output = self._layer(shared_experts_input)
current_stream().wait_stream(self._stream)
self._input_ready_event[idx].wait(self._stream)
self._output[idx] = self._layer(shared_experts_input)
self._output_ready_event[idx].record(self._stream)
return True
return output
def wait(self) -> None:
"""Block the main stream until `maybe_forward_async` output is ready."""
assert self._stream is not None
self._output_ready_event[self._output_idx].wait(current_stream())
@property
def _output_idx(self) -> int:
@@ -174,11 +181,6 @@ class SharedExperts(torch.nn.Module):
assert self._output[self._output_idx] is None
if order == SharedExpertsOrder.MULTI_STREAM_OVERLAPPED:
self._output[self._output_idx] = self._run_in_aux_stream(
shared_experts_input
)
else:
self._output[self._output_idx] = self._layer(shared_experts_input)
self._output[self._output_idx] = self._layer(shared_experts_input)
assert self._output[self._output_idx] is not None