diff --git a/vllm/model_executor/layers/fused_moe/runner/moe_runner.py b/vllm/model_executor/layers/fused_moe/runner/moe_runner.py index 1c34da3b67..aaf9ba6142 100644 --- a/vllm/model_executor/layers/fused_moe/runner/moe_runner.py +++ b/vllm/model_executor/layers/fused_moe/runner/moe_runner.py @@ -573,17 +573,12 @@ class MoERunner(MoERunnerInterface): router_logits: torch.Tensor, shared_experts_input: torch.Tensor | None, input_ids: torch.Tensor | None = None, - shared_experts_overlapping: bool = False, ) -> tuple[torch.Tensor | None, torch.Tensor]: """Run expert routing and the fused MoE kernel via the quant method. Orchestrates shared expert execution (before/after), expert selection via the router, and the actual fused MoE computation. Returns (shared_expert_output, fused_expert_output). - - `shared_experts_overlapping` should be True only if using multi-stream - overlap. Then the shared expert was already launched in a separate - stream, so the results only have to be awaited here. """ self._maybe_apply_shared_experts( shared_experts_input, SharedExpertsOrder.NO_OVERLAP @@ -613,9 +608,10 @@ class MoERunner(MoERunnerInterface): shared_experts_input=shared_experts_input, ) - if shared_experts_overlapping: - assert self._shared_experts is not None - self._shared_experts.wait() + self._maybe_apply_shared_experts( + shared_experts_input, + SharedExpertsOrder.MULTI_STREAM_OVERLAPPED, + ) return ( self._shared_experts.output if self._shared_experts is not None else None, @@ -637,6 +633,18 @@ class MoERunner(MoERunnerInterface): else nullcontext() ) + def _maybe_sync_shared_experts_stream( + self, + shared_experts_input: torch.Tensor | None, + ): + # If router/gate provided, then apply it here. + # (Note: This code runs only when "overlapped mode" is on to allow + # parallel execution of shared experts with the RoutedExperts via + # separate cuda stream) + if self._shared_experts is not None: + assert shared_experts_input is not None + self._shared_experts.maybe_sync_shared_experts_stream(shared_experts_input) + def _maybe_add_zero_expert_output( self, result: torch.Tensor, @@ -841,13 +849,8 @@ class MoERunner(MoERunnerInterface): # TODO(bnell): this can be removed after MK migration is complete. self.routed_experts._ensure_moe_quant_config_init() - # If using multi-stream overlap for shared experts, we must launch it - # before routed expert dispatch. - shared_experts_overlapping = False - if self._shared_experts is not None: - shared_experts_overlapping = self._shared_experts.maybe_forward_async( - shared_experts_input - ) + # Sync aux and main stream for shared expert multi-stream overlap. + self._maybe_sync_shared_experts_stream(shared_experts_input) # If the Runner holds the gate, apply it after the stream sync, # so it can run overlapped with the @@ -873,7 +876,6 @@ class MoERunner(MoERunnerInterface): router_logits=router_logits, shared_experts_input=shared_experts_input, input_ids=input_ids, - shared_experts_overlapping=shared_experts_overlapping, ) return self._maybe_combine( diff --git a/vllm/model_executor/layers/fused_moe/runner/shared_experts.py b/vllm/model_executor/layers/fused_moe/runner/shared_experts.py index 992b2f8368..17faeffd29 100644 --- a/vllm/model_executor/layers/fused_moe/runner/shared_experts.py +++ b/vllm/model_executor/layers/fused_moe/runner/shared_experts.py @@ -71,11 +71,6 @@ class SharedExperts(torch.nn.Module): if self._stream is not None: logger.debug_once("Enabled separate cuda stream for MoE shared_experts") - if self._stream is not None: - # One pair per DBO ubatch id. - self._input_ready_event = [torch.cuda.Event(), torch.cuda.Event()] - self._output_ready_event = [torch.cuda.Event(), torch.cuda.Event()] - # TODO(bnell): Hack for elastic_ep. Get rid of this def _set_moe_config(self, new_moe_config: FusedMoEConfig): self.moe_config = new_moe_config @@ -101,14 +96,6 @@ class SharedExperts(torch.nn.Module): and parallel_config.all2all_backend not in _EPLB_OVERLAP_SAFE_BACKENDS ) or parallel_config.use_fi_nvl_two_sided_kernels - @property - def _should_enable_stream_overlap_heuristic(self) -> bool: - # On ROCm, empirically it's shown that only DPA deployments benefit from - # multi-stream shared experts - if not current_platform.is_rocm(): - return True - return self._moe_config.moe_parallel_config.dp_size > 1 - def _determine_shared_experts_order( self, hidden_states: torch.Tensor, @@ -120,11 +107,10 @@ class SharedExperts(torch.nn.Module): return SharedExpertsOrder.MK_INTERNAL_OVERLAPPED should_run_shared_in_aux_stream = ( - current_platform.is_cuda_alike() + current_platform.is_cuda() and self._stream is not None and hidden_states.shape[0] <= envs.VLLM_SHARED_EXPERTS_STREAM_TOKEN_THRESHOLD - and self._should_enable_stream_overlap_heuristic ) if should_run_shared_in_aux_stream: @@ -132,31 +118,38 @@ class SharedExperts(torch.nn.Module): else: return SharedExpertsOrder.NO_OVERLAP - def maybe_forward_async(self, shared_experts_input: torch.Tensor) -> bool: - """Enqueue shared experts on the aux stream without waiting for them. + def maybe_sync_shared_experts_stream( + self, + shared_experts_input: torch.Tensor, + ): + experts_order = self._determine_shared_experts_order(shared_experts_input) - Returns true if the shared experts were enqueued, false otherwise. Call - `wait` to wait for the shared experts to finish if this returns true. - """ - if ( - self._determine_shared_experts_order(shared_experts_input) - != SharedExpertsOrder.MULTI_STREAM_OVERLAPPED - ): - return False - assert self._stream is not None - idx = self._output_idx - assert self._output[idx] is None - self._input_ready_event[idx].record(current_stream()) + if experts_order == SharedExpertsOrder.MULTI_STREAM_OVERLAPPED: + assert self._stream is not None + + # Record that the clone will be used by shared_experts_stream + # to avoid gc issue from deallocation of hidden_states_clone + # For more details: https://docs.pytorch.org/docs/stable/generated/torch.Tensor.record_stream.html # noqa: E501 + # NOTE: We don't need shared_output.record_stream(current_stream()) + # because we synch the streams before using shared_output. + shared_experts_input.record_stream(self._stream) + + # Mark sync start point for the aux stream since we will + # run in parallel with router/gate. + self._stream.wait_stream(current_stream()) + + def _run_in_aux_stream( + self, + shared_experts_input: torch.Tensor, + ) -> torch.Tensor: + # TODO: assert that maybe_sync_shared_experts_stream has been called. + + # Run shared experts in parallel on a separate stream. with torch.cuda.stream(self._stream): - self._input_ready_event[idx].wait(self._stream) - self._output[idx] = self._layer(shared_experts_input) - self._output_ready_event[idx].record(self._stream) - return True + output = self._layer(shared_experts_input) + current_stream().wait_stream(self._stream) - def wait(self) -> None: - """Block the main stream until `maybe_forward_async` output is ready.""" - assert self._stream is not None - self._output_ready_event[self._output_idx].wait(current_stream()) + return output @property def _output_idx(self) -> int: @@ -181,6 +174,11 @@ class SharedExperts(torch.nn.Module): assert self._output[self._output_idx] is None - self._output[self._output_idx] = self._layer(shared_experts_input) + if order == SharedExpertsOrder.MULTI_STREAM_OVERLAPPED: + self._output[self._output_idx] = self._run_in_aux_stream( + shared_experts_input + ) + else: + self._output[self._output_idx] = self._layer(shared_experts_input) assert self._output[self._output_idx] is not None