Revert "[Perf][ROCm] Dual-stream decode with hipgraphs" (#52024)
Signed-off-by: simondanielsson <simon.danielsson99@hotmail.com> Co-authored-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
co-authored by
Andreas Karatzas
parent
2ac1f683f1
commit
3d204dfdaa
@@ -573,17 +573,12 @@ class MoERunner(MoERunnerInterface):
|
||||
router_logits: torch.Tensor,
|
||||
shared_experts_input: torch.Tensor | None,
|
||||
input_ids: torch.Tensor | None = None,
|
||||
shared_experts_overlapping: bool = False,
|
||||
) -> tuple[torch.Tensor | None, torch.Tensor]:
|
||||
"""Run expert routing and the fused MoE kernel via the quant method.
|
||||
|
||||
Orchestrates shared expert execution (before/after), expert selection
|
||||
via the router, and the actual fused MoE computation. Returns
|
||||
(shared_expert_output, fused_expert_output).
|
||||
|
||||
`shared_experts_overlapping` should be True only if using multi-stream
|
||||
overlap. Then the shared expert was already launched in a separate
|
||||
stream, so the results only have to be awaited here.
|
||||
"""
|
||||
self._maybe_apply_shared_experts(
|
||||
shared_experts_input, SharedExpertsOrder.NO_OVERLAP
|
||||
@@ -613,9 +608,10 @@ class MoERunner(MoERunnerInterface):
|
||||
shared_experts_input=shared_experts_input,
|
||||
)
|
||||
|
||||
if shared_experts_overlapping:
|
||||
assert self._shared_experts is not None
|
||||
self._shared_experts.wait()
|
||||
self._maybe_apply_shared_experts(
|
||||
shared_experts_input,
|
||||
SharedExpertsOrder.MULTI_STREAM_OVERLAPPED,
|
||||
)
|
||||
|
||||
return (
|
||||
self._shared_experts.output if self._shared_experts is not None else None,
|
||||
@@ -637,6 +633,18 @@ class MoERunner(MoERunnerInterface):
|
||||
else nullcontext()
|
||||
)
|
||||
|
||||
def _maybe_sync_shared_experts_stream(
|
||||
self,
|
||||
shared_experts_input: torch.Tensor | None,
|
||||
):
|
||||
# If router/gate provided, then apply it here.
|
||||
# (Note: This code runs only when "overlapped mode" is on to allow
|
||||
# parallel execution of shared experts with the RoutedExperts via
|
||||
# separate cuda stream)
|
||||
if self._shared_experts is not None:
|
||||
assert shared_experts_input is not None
|
||||
self._shared_experts.maybe_sync_shared_experts_stream(shared_experts_input)
|
||||
|
||||
def _maybe_add_zero_expert_output(
|
||||
self,
|
||||
result: torch.Tensor,
|
||||
@@ -841,13 +849,8 @@ class MoERunner(MoERunnerInterface):
|
||||
# TODO(bnell): this can be removed after MK migration is complete.
|
||||
self.routed_experts._ensure_moe_quant_config_init()
|
||||
|
||||
# If using multi-stream overlap for shared experts, we must launch it
|
||||
# before routed expert dispatch.
|
||||
shared_experts_overlapping = False
|
||||
if self._shared_experts is not None:
|
||||
shared_experts_overlapping = self._shared_experts.maybe_forward_async(
|
||||
shared_experts_input
|
||||
)
|
||||
# Sync aux and main stream for shared expert multi-stream overlap.
|
||||
self._maybe_sync_shared_experts_stream(shared_experts_input)
|
||||
|
||||
# If the Runner holds the gate, apply it after the stream sync,
|
||||
# so it can run overlapped with the
|
||||
@@ -873,7 +876,6 @@ class MoERunner(MoERunnerInterface):
|
||||
router_logits=router_logits,
|
||||
shared_experts_input=shared_experts_input,
|
||||
input_ids=input_ids,
|
||||
shared_experts_overlapping=shared_experts_overlapping,
|
||||
)
|
||||
|
||||
return self._maybe_combine(
|
||||
|
||||
@@ -71,11 +71,6 @@ class SharedExperts(torch.nn.Module):
|
||||
if self._stream is not None:
|
||||
logger.debug_once("Enabled separate cuda stream for MoE shared_experts")
|
||||
|
||||
if self._stream is not None:
|
||||
# One pair per DBO ubatch id.
|
||||
self._input_ready_event = [torch.cuda.Event(), torch.cuda.Event()]
|
||||
self._output_ready_event = [torch.cuda.Event(), torch.cuda.Event()]
|
||||
|
||||
# TODO(bnell): Hack for elastic_ep. Get rid of this
|
||||
def _set_moe_config(self, new_moe_config: FusedMoEConfig):
|
||||
self.moe_config = new_moe_config
|
||||
@@ -101,14 +96,6 @@ class SharedExperts(torch.nn.Module):
|
||||
and parallel_config.all2all_backend not in _EPLB_OVERLAP_SAFE_BACKENDS
|
||||
) or parallel_config.use_fi_nvl_two_sided_kernels
|
||||
|
||||
@property
|
||||
def _should_enable_stream_overlap_heuristic(self) -> bool:
|
||||
# On ROCm, empirically it's shown that only DPA deployments benefit from
|
||||
# multi-stream shared experts
|
||||
if not current_platform.is_rocm():
|
||||
return True
|
||||
return self._moe_config.moe_parallel_config.dp_size > 1
|
||||
|
||||
def _determine_shared_experts_order(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
@@ -120,11 +107,10 @@ class SharedExperts(torch.nn.Module):
|
||||
return SharedExpertsOrder.MK_INTERNAL_OVERLAPPED
|
||||
|
||||
should_run_shared_in_aux_stream = (
|
||||
current_platform.is_cuda_alike()
|
||||
current_platform.is_cuda()
|
||||
and self._stream is not None
|
||||
and hidden_states.shape[0]
|
||||
<= envs.VLLM_SHARED_EXPERTS_STREAM_TOKEN_THRESHOLD
|
||||
and self._should_enable_stream_overlap_heuristic
|
||||
)
|
||||
|
||||
if should_run_shared_in_aux_stream:
|
||||
@@ -132,31 +118,38 @@ class SharedExperts(torch.nn.Module):
|
||||
else:
|
||||
return SharedExpertsOrder.NO_OVERLAP
|
||||
|
||||
def maybe_forward_async(self, shared_experts_input: torch.Tensor) -> bool:
|
||||
"""Enqueue shared experts on the aux stream without waiting for them.
|
||||
def maybe_sync_shared_experts_stream(
|
||||
self,
|
||||
shared_experts_input: torch.Tensor,
|
||||
):
|
||||
experts_order = self._determine_shared_experts_order(shared_experts_input)
|
||||
|
||||
Returns true if the shared experts were enqueued, false otherwise. Call
|
||||
`wait` to wait for the shared experts to finish if this returns true.
|
||||
"""
|
||||
if (
|
||||
self._determine_shared_experts_order(shared_experts_input)
|
||||
!= SharedExpertsOrder.MULTI_STREAM_OVERLAPPED
|
||||
):
|
||||
return False
|
||||
assert self._stream is not None
|
||||
idx = self._output_idx
|
||||
assert self._output[idx] is None
|
||||
self._input_ready_event[idx].record(current_stream())
|
||||
if experts_order == SharedExpertsOrder.MULTI_STREAM_OVERLAPPED:
|
||||
assert self._stream is not None
|
||||
|
||||
# Record that the clone will be used by shared_experts_stream
|
||||
# to avoid gc issue from deallocation of hidden_states_clone
|
||||
# For more details: https://docs.pytorch.org/docs/stable/generated/torch.Tensor.record_stream.html # noqa: E501
|
||||
# NOTE: We don't need shared_output.record_stream(current_stream())
|
||||
# because we synch the streams before using shared_output.
|
||||
shared_experts_input.record_stream(self._stream)
|
||||
|
||||
# Mark sync start point for the aux stream since we will
|
||||
# run in parallel with router/gate.
|
||||
self._stream.wait_stream(current_stream())
|
||||
|
||||
def _run_in_aux_stream(
|
||||
self,
|
||||
shared_experts_input: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
# TODO: assert that maybe_sync_shared_experts_stream has been called.
|
||||
|
||||
# Run shared experts in parallel on a separate stream.
|
||||
with torch.cuda.stream(self._stream):
|
||||
self._input_ready_event[idx].wait(self._stream)
|
||||
self._output[idx] = self._layer(shared_experts_input)
|
||||
self._output_ready_event[idx].record(self._stream)
|
||||
return True
|
||||
output = self._layer(shared_experts_input)
|
||||
current_stream().wait_stream(self._stream)
|
||||
|
||||
def wait(self) -> None:
|
||||
"""Block the main stream until `maybe_forward_async` output is ready."""
|
||||
assert self._stream is not None
|
||||
self._output_ready_event[self._output_idx].wait(current_stream())
|
||||
return output
|
||||
|
||||
@property
|
||||
def _output_idx(self) -> int:
|
||||
@@ -181,6 +174,11 @@ class SharedExperts(torch.nn.Module):
|
||||
|
||||
assert self._output[self._output_idx] is None
|
||||
|
||||
self._output[self._output_idx] = self._layer(shared_experts_input)
|
||||
if order == SharedExpertsOrder.MULTI_STREAM_OVERLAPPED:
|
||||
self._output[self._output_idx] = self._run_in_aux_stream(
|
||||
shared_experts_input
|
||||
)
|
||||
else:
|
||||
self._output[self._output_idx] = self._layer(shared_experts_input)
|
||||
|
||||
assert self._output[self._output_idx] is not None
|
||||
|
||||
Reference in New Issue
Block a user