[Perf][ROCm] Dual-stream decode with hipgraphs (#52033)
Signed-off-by: simondanielsson <simon.danielsson99@hotmail.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: Shanshan Shen <467638484@qq.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Shanshan Shen
parent
4f78a8fdd0
commit
2c7d7dd64a
@@ -283,11 +283,24 @@ class MoERunner(MoERunnerInterface):
|
|||||||
self._shared_experts: SharedExperts | None = None
|
self._shared_experts: SharedExperts | None = None
|
||||||
if shared_experts is not None:
|
if shared_experts is not None:
|
||||||
can_overlap = lambda: self._quant_method.mk_can_overlap_shared_experts
|
can_overlap = lambda: self._quant_method.mk_can_overlap_shared_experts
|
||||||
|
# When unquantized, shared expert inputs alias the hidden states,
|
||||||
|
# which can lead to race condition in multi-stream mode. Quantized
|
||||||
|
# routed experts copy the input into a fresh buffer first, breaking
|
||||||
|
# the alias, so overlap is safe. Only observed on ROCm.
|
||||||
|
routed_input_is_quantized = lambda: (
|
||||||
|
self.routed_experts.quant_method.moe_quant_config is not None
|
||||||
|
and self.routed_experts.quant_method.moe_quant_config.quant_dtype
|
||||||
|
is not None
|
||||||
|
)
|
||||||
|
is_multistream_safe = lambda: (
|
||||||
|
not current_platform.is_rocm() or routed_input_is_quantized()
|
||||||
|
)
|
||||||
self._shared_experts = SharedExperts(
|
self._shared_experts = SharedExperts(
|
||||||
shared_experts,
|
shared_experts,
|
||||||
moe_config=moe_config,
|
moe_config=moe_config,
|
||||||
enable_dbo=enable_dbo,
|
enable_dbo=enable_dbo,
|
||||||
mk_can_overlap_shared_experts=can_overlap,
|
mk_can_overlap_shared_experts=can_overlap,
|
||||||
|
is_multistream_safe=is_multistream_safe,
|
||||||
)
|
)
|
||||||
|
|
||||||
# Needed for string -> MoERunner layer lookup in custom ops.
|
# Needed for string -> MoERunner layer lookup in custom ops.
|
||||||
@@ -582,12 +595,17 @@ class MoERunner(MoERunnerInterface):
|
|||||||
router_logits: torch.Tensor,
|
router_logits: torch.Tensor,
|
||||||
shared_experts_input: torch.Tensor | None,
|
shared_experts_input: torch.Tensor | None,
|
||||||
input_ids: torch.Tensor | None = None,
|
input_ids: torch.Tensor | None = None,
|
||||||
|
shared_experts_overlapping: bool = False,
|
||||||
) -> tuple[torch.Tensor | None, torch.Tensor | UnfinalizedMoEOutput]:
|
) -> tuple[torch.Tensor | None, torch.Tensor | UnfinalizedMoEOutput]:
|
||||||
"""Run expert routing and the fused MoE kernel via the quant method.
|
"""Run expert routing and the fused MoE kernel via the quant method.
|
||||||
|
|
||||||
Orchestrates shared expert execution (before/after), expert selection
|
Orchestrates shared expert execution (before/after), expert selection
|
||||||
via the router, and the actual fused MoE computation. Returns
|
via the router, and the actual fused MoE computation. Returns
|
||||||
(shared_expert_output, fused_expert_output).
|
(shared_expert_output, fused_expert_output).
|
||||||
|
|
||||||
|
`shared_experts_overlapping` should be True only if using multi-stream
|
||||||
|
overlap. Then the shared expert was already launched in a separate
|
||||||
|
stream, so the results only have to be awaited here.
|
||||||
"""
|
"""
|
||||||
self._maybe_apply_shared_experts(
|
self._maybe_apply_shared_experts(
|
||||||
shared_experts_input, SharedExpertsOrder.NO_OVERLAP
|
shared_experts_input, SharedExpertsOrder.NO_OVERLAP
|
||||||
@@ -617,10 +635,9 @@ class MoERunner(MoERunnerInterface):
|
|||||||
shared_experts_input=shared_experts_input,
|
shared_experts_input=shared_experts_input,
|
||||||
)
|
)
|
||||||
|
|
||||||
self._maybe_apply_shared_experts(
|
if shared_experts_overlapping:
|
||||||
shared_experts_input,
|
assert self._shared_experts is not None
|
||||||
SharedExpertsOrder.MULTI_STREAM_OVERLAPPED,
|
self._shared_experts.wait()
|
||||||
)
|
|
||||||
|
|
||||||
return (
|
return (
|
||||||
self._shared_experts.output if self._shared_experts is not None else None,
|
self._shared_experts.output if self._shared_experts is not None else None,
|
||||||
@@ -642,18 +659,6 @@ class MoERunner(MoERunnerInterface):
|
|||||||
else nullcontext()
|
else nullcontext()
|
||||||
)
|
)
|
||||||
|
|
||||||
def _maybe_sync_shared_experts_stream(
|
|
||||||
self,
|
|
||||||
shared_experts_input: torch.Tensor | None,
|
|
||||||
):
|
|
||||||
# If router/gate provided, then apply it here.
|
|
||||||
# (Note: This code runs only when "overlapped mode" is on to allow
|
|
||||||
# parallel execution of shared experts with the RoutedExperts via
|
|
||||||
# separate cuda stream)
|
|
||||||
if self._shared_experts is not None:
|
|
||||||
assert shared_experts_input is not None
|
|
||||||
self._shared_experts.maybe_sync_shared_experts_stream(shared_experts_input)
|
|
||||||
|
|
||||||
def _maybe_add_zero_expert_output(
|
def _maybe_add_zero_expert_output(
|
||||||
self,
|
self,
|
||||||
result: torch.Tensor,
|
result: torch.Tensor,
|
||||||
@@ -878,8 +883,13 @@ class MoERunner(MoERunnerInterface):
|
|||||||
# TODO(bnell): this can be removed after MK migration is complete.
|
# TODO(bnell): this can be removed after MK migration is complete.
|
||||||
self.routed_experts._ensure_moe_quant_config_init()
|
self.routed_experts._ensure_moe_quant_config_init()
|
||||||
|
|
||||||
# Sync aux and main stream for shared expert multi-stream overlap.
|
# If using multi-stream overlap for shared experts, we must launch it
|
||||||
self._maybe_sync_shared_experts_stream(shared_experts_input)
|
# before routed expert dispatch.
|
||||||
|
shared_experts_overlapping = False
|
||||||
|
if self._shared_experts is not None:
|
||||||
|
shared_experts_overlapping = self._shared_experts.maybe_forward_async(
|
||||||
|
shared_experts_input
|
||||||
|
)
|
||||||
|
|
||||||
# If the Runner holds the gate, apply it after the stream sync,
|
# If the Runner holds the gate, apply it after the stream sync,
|
||||||
# so it can run overlapped with the
|
# so it can run overlapped with the
|
||||||
@@ -905,6 +915,7 @@ class MoERunner(MoERunnerInterface):
|
|||||||
router_logits=router_logits,
|
router_logits=router_logits,
|
||||||
shared_experts_input=shared_experts_input,
|
shared_experts_input=shared_experts_input,
|
||||||
input_ids=input_ids,
|
input_ids=input_ids,
|
||||||
|
shared_experts_overlapping=shared_experts_overlapping,
|
||||||
)
|
)
|
||||||
|
|
||||||
return self._maybe_combine(
|
return self._maybe_combine(
|
||||||
|
|||||||
@@ -43,6 +43,7 @@ class SharedExperts(torch.nn.Module):
|
|||||||
moe_config: FusedMoEConfig,
|
moe_config: FusedMoEConfig,
|
||||||
enable_dbo: bool,
|
enable_dbo: bool,
|
||||||
mk_can_overlap_shared_experts: Callable[[], bool],
|
mk_can_overlap_shared_experts: Callable[[], bool],
|
||||||
|
is_multistream_safe: Callable[[], bool],
|
||||||
):
|
):
|
||||||
super().__init__()
|
super().__init__()
|
||||||
|
|
||||||
@@ -57,6 +58,10 @@ class SharedExperts(torch.nn.Module):
|
|||||||
|
|
||||||
self._mk_can_overlap_shared_experts = mk_can_overlap_shared_experts
|
self._mk_can_overlap_shared_experts = mk_can_overlap_shared_experts
|
||||||
|
|
||||||
|
# Might not be safe to run multi-stream mode if routed and shared experts
|
||||||
|
# alias the same inputs
|
||||||
|
self._is_multistream_safe = is_multistream_safe
|
||||||
|
|
||||||
# Allow disabling of the separate shared experts stream for
|
# Allow disabling of the separate shared experts stream for
|
||||||
# debug purposes.
|
# debug purposes.
|
||||||
# TODO: Remove this after more extensive testings with TP/DP
|
# TODO: Remove this after more extensive testings with TP/DP
|
||||||
@@ -65,11 +70,12 @@ class SharedExperts(torch.nn.Module):
|
|||||||
logger.debug_once("Disabling MoE shared_experts cuda stream")
|
logger.debug_once("Disabling MoE shared_experts cuda stream")
|
||||||
self._stream = None
|
self._stream = None
|
||||||
else:
|
else:
|
||||||
# TODO(rob): enable shared expert overlap with non-cuda-alike.
|
|
||||||
# aux_stream() returns None on non-cuda-alike platforms.
|
|
||||||
self._stream = aux_stream()
|
self._stream = aux_stream()
|
||||||
if self._stream is not None:
|
if self._stream is not None:
|
||||||
logger.debug_once("Enabled separate cuda stream for MoE shared_experts")
|
logger.debug_once("Enabled separate cuda stream for MoE shared_experts")
|
||||||
|
# One pair per DBO ubatch id to sync aux and main stream.
|
||||||
|
self._input_ready_event = [torch.cuda.Event(), torch.cuda.Event()]
|
||||||
|
self._output_ready_event = [torch.cuda.Event(), torch.cuda.Event()]
|
||||||
|
|
||||||
# TODO(bnell): Hack for elastic_ep. Get rid of this
|
# TODO(bnell): Hack for elastic_ep. Get rid of this
|
||||||
def _set_moe_config(self, new_moe_config: FusedMoEConfig):
|
def _set_moe_config(self, new_moe_config: FusedMoEConfig):
|
||||||
@@ -106,11 +112,19 @@ class SharedExperts(torch.nn.Module):
|
|||||||
if self._mk_can_overlap_shared_experts():
|
if self._mk_can_overlap_shared_experts():
|
||||||
return SharedExpertsOrder.MK_INTERNAL_OVERLAPPED
|
return SharedExpertsOrder.MK_INTERNAL_OVERLAPPED
|
||||||
|
|
||||||
|
# On ROCm, empirically only DP-only deployments benefit from the overlap.
|
||||||
|
overlap_is_beneficial = not current_platform.is_rocm() or (
|
||||||
|
self._moe_config.moe_parallel_config.dp_size > 1
|
||||||
|
and self._moe_config.moe_parallel_config.tp_size == 1
|
||||||
|
)
|
||||||
|
|
||||||
should_run_shared_in_aux_stream = (
|
should_run_shared_in_aux_stream = (
|
||||||
current_platform.is_cuda()
|
current_platform.is_cuda_alike()
|
||||||
and self._stream is not None
|
and self._stream is not None
|
||||||
and hidden_states.shape[0]
|
and hidden_states.shape[0]
|
||||||
<= envs.VLLM_SHARED_EXPERTS_STREAM_TOKEN_THRESHOLD
|
<= envs.VLLM_SHARED_EXPERTS_STREAM_TOKEN_THRESHOLD
|
||||||
|
and overlap_is_beneficial
|
||||||
|
and self._is_multistream_safe()
|
||||||
)
|
)
|
||||||
|
|
||||||
if should_run_shared_in_aux_stream:
|
if should_run_shared_in_aux_stream:
|
||||||
@@ -118,38 +132,31 @@ class SharedExperts(torch.nn.Module):
|
|||||||
else:
|
else:
|
||||||
return SharedExpertsOrder.NO_OVERLAP
|
return SharedExpertsOrder.NO_OVERLAP
|
||||||
|
|
||||||
def maybe_sync_shared_experts_stream(
|
def maybe_forward_async(self, shared_experts_input: torch.Tensor) -> bool:
|
||||||
self,
|
"""Enqueue shared experts on the aux stream without waiting for them.
|
||||||
shared_experts_input: torch.Tensor,
|
|
||||||
):
|
|
||||||
experts_order = self._determine_shared_experts_order(shared_experts_input)
|
|
||||||
|
|
||||||
if experts_order == SharedExpertsOrder.MULTI_STREAM_OVERLAPPED:
|
Returns true if the shared experts were enqueued, false otherwise. Call
|
||||||
assert self._stream is not None
|
`wait` to wait for the shared experts to finish if this returns true.
|
||||||
|
"""
|
||||||
# Record that the clone will be used by shared_experts_stream
|
if (
|
||||||
# to avoid gc issue from deallocation of hidden_states_clone
|
self._determine_shared_experts_order(shared_experts_input)
|
||||||
# For more details: https://docs.pytorch.org/docs/stable/generated/torch.Tensor.record_stream.html # noqa: E501
|
!= SharedExpertsOrder.MULTI_STREAM_OVERLAPPED
|
||||||
# NOTE: We don't need shared_output.record_stream(current_stream())
|
):
|
||||||
# because we synch the streams before using shared_output.
|
return False
|
||||||
shared_experts_input.record_stream(self._stream)
|
assert self._stream is not None
|
||||||
|
idx = self._output_idx
|
||||||
# Mark sync start point for the aux stream since we will
|
assert self._output[idx] is None
|
||||||
# run in parallel with router/gate.
|
self._input_ready_event[idx].record(current_stream())
|
||||||
self._stream.wait_stream(current_stream())
|
|
||||||
|
|
||||||
def _run_in_aux_stream(
|
|
||||||
self,
|
|
||||||
shared_experts_input: torch.Tensor,
|
|
||||||
) -> torch.Tensor:
|
|
||||||
# TODO: assert that maybe_sync_shared_experts_stream has been called.
|
|
||||||
|
|
||||||
# Run shared experts in parallel on a separate stream.
|
|
||||||
with torch.cuda.stream(self._stream):
|
with torch.cuda.stream(self._stream):
|
||||||
output = self._layer(shared_experts_input)
|
self._input_ready_event[idx].wait(self._stream)
|
||||||
current_stream().wait_stream(self._stream)
|
self._output[idx] = self._layer(shared_experts_input)
|
||||||
|
self._output_ready_event[idx].record(self._stream)
|
||||||
|
return True
|
||||||
|
|
||||||
return output
|
def wait(self) -> None:
|
||||||
|
"""Block the main stream until `maybe_forward_async` output is ready."""
|
||||||
|
assert self._stream is not None
|
||||||
|
self._output_ready_event[self._output_idx].wait(current_stream())
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def _output_idx(self) -> int:
|
def _output_idx(self) -> int:
|
||||||
@@ -174,11 +181,6 @@ class SharedExperts(torch.nn.Module):
|
|||||||
|
|
||||||
assert self._output[self._output_idx] is None
|
assert self._output[self._output_idx] is None
|
||||||
|
|
||||||
if order == SharedExpertsOrder.MULTI_STREAM_OVERLAPPED:
|
self._output[self._output_idx] = self._layer(shared_experts_input)
|
||||||
self._output[self._output_idx] = self._run_in_aux_stream(
|
|
||||||
shared_experts_input
|
|
||||||
)
|
|
||||||
else:
|
|
||||||
self._output[self._output_idx] = self._layer(shared_experts_input)
|
|
||||||
|
|
||||||
assert self._output[self._output_idx] is not None
|
assert self._output[self._output_idx] is not None
|
||||||
|
|||||||
Reference in New Issue
Block a user