diff --git a/vllm/model_executor/layers/fused_moe/runner/moe_runner.py b/vllm/model_executor/layers/fused_moe/runner/moe_runner.py index aaf9ba6142..1c34da3b67 100644 --- a/vllm/model_executor/layers/fused_moe/runner/moe_runner.py +++ b/vllm/model_executor/layers/fused_moe/runner/moe_runner.py @@ -573,12 +573,17 @@ class MoERunner(MoERunnerInterface): router_logits: torch.Tensor, shared_experts_input: torch.Tensor | None, input_ids: torch.Tensor | None = None, + shared_experts_overlapping: bool = False, ) -> tuple[torch.Tensor | None, torch.Tensor]: """Run expert routing and the fused MoE kernel via the quant method. Orchestrates shared expert execution (before/after), expert selection via the router, and the actual fused MoE computation. Returns (shared_expert_output, fused_expert_output). + + `shared_experts_overlapping` should be True only if using multi-stream + overlap. Then the shared expert was already launched in a separate + stream, so the results only have to be awaited here. """ self._maybe_apply_shared_experts( shared_experts_input, SharedExpertsOrder.NO_OVERLAP @@ -608,10 +613,9 @@ class MoERunner(MoERunnerInterface): shared_experts_input=shared_experts_input, ) - self._maybe_apply_shared_experts( - shared_experts_input, - SharedExpertsOrder.MULTI_STREAM_OVERLAPPED, - ) + if shared_experts_overlapping: + assert self._shared_experts is not None + self._shared_experts.wait() return ( self._shared_experts.output if self._shared_experts is not None else None, @@ -633,18 +637,6 @@ class MoERunner(MoERunnerInterface): else nullcontext() ) - def _maybe_sync_shared_experts_stream( - self, - shared_experts_input: torch.Tensor | None, - ): - # If router/gate provided, then apply it here. - # (Note: This code runs only when "overlapped mode" is on to allow - # parallel execution of shared experts with the RoutedExperts via - # separate cuda stream) - if self._shared_experts is not None: - assert shared_experts_input is not None - self._shared_experts.maybe_sync_shared_experts_stream(shared_experts_input) - def _maybe_add_zero_expert_output( self, result: torch.Tensor, @@ -849,8 +841,13 @@ class MoERunner(MoERunnerInterface): # TODO(bnell): this can be removed after MK migration is complete. self.routed_experts._ensure_moe_quant_config_init() - # Sync aux and main stream for shared expert multi-stream overlap. - self._maybe_sync_shared_experts_stream(shared_experts_input) + # If using multi-stream overlap for shared experts, we must launch it + # before routed expert dispatch. + shared_experts_overlapping = False + if self._shared_experts is not None: + shared_experts_overlapping = self._shared_experts.maybe_forward_async( + shared_experts_input + ) # If the Runner holds the gate, apply it after the stream sync, # so it can run overlapped with the @@ -876,6 +873,7 @@ class MoERunner(MoERunnerInterface): router_logits=router_logits, shared_experts_input=shared_experts_input, input_ids=input_ids, + shared_experts_overlapping=shared_experts_overlapping, ) return self._maybe_combine( diff --git a/vllm/model_executor/layers/fused_moe/runner/shared_experts.py b/vllm/model_executor/layers/fused_moe/runner/shared_experts.py index 17faeffd29..992b2f8368 100644 --- a/vllm/model_executor/layers/fused_moe/runner/shared_experts.py +++ b/vllm/model_executor/layers/fused_moe/runner/shared_experts.py @@ -71,6 +71,11 @@ class SharedExperts(torch.nn.Module): if self._stream is not None: logger.debug_once("Enabled separate cuda stream for MoE shared_experts") + if self._stream is not None: + # One pair per DBO ubatch id. + self._input_ready_event = [torch.cuda.Event(), torch.cuda.Event()] + self._output_ready_event = [torch.cuda.Event(), torch.cuda.Event()] + # TODO(bnell): Hack for elastic_ep. Get rid of this def _set_moe_config(self, new_moe_config: FusedMoEConfig): self.moe_config = new_moe_config @@ -96,6 +101,14 @@ class SharedExperts(torch.nn.Module): and parallel_config.all2all_backend not in _EPLB_OVERLAP_SAFE_BACKENDS ) or parallel_config.use_fi_nvl_two_sided_kernels + @property + def _should_enable_stream_overlap_heuristic(self) -> bool: + # On ROCm, empirically it's shown that only DPA deployments benefit from + # multi-stream shared experts + if not current_platform.is_rocm(): + return True + return self._moe_config.moe_parallel_config.dp_size > 1 + def _determine_shared_experts_order( self, hidden_states: torch.Tensor, @@ -107,10 +120,11 @@ class SharedExperts(torch.nn.Module): return SharedExpertsOrder.MK_INTERNAL_OVERLAPPED should_run_shared_in_aux_stream = ( - current_platform.is_cuda() + current_platform.is_cuda_alike() and self._stream is not None and hidden_states.shape[0] <= envs.VLLM_SHARED_EXPERTS_STREAM_TOKEN_THRESHOLD + and self._should_enable_stream_overlap_heuristic ) if should_run_shared_in_aux_stream: @@ -118,38 +132,31 @@ class SharedExperts(torch.nn.Module): else: return SharedExpertsOrder.NO_OVERLAP - def maybe_sync_shared_experts_stream( - self, - shared_experts_input: torch.Tensor, - ): - experts_order = self._determine_shared_experts_order(shared_experts_input) + def maybe_forward_async(self, shared_experts_input: torch.Tensor) -> bool: + """Enqueue shared experts on the aux stream without waiting for them. - if experts_order == SharedExpertsOrder.MULTI_STREAM_OVERLAPPED: - assert self._stream is not None - - # Record that the clone will be used by shared_experts_stream - # to avoid gc issue from deallocation of hidden_states_clone - # For more details: https://docs.pytorch.org/docs/stable/generated/torch.Tensor.record_stream.html # noqa: E501 - # NOTE: We don't need shared_output.record_stream(current_stream()) - # because we synch the streams before using shared_output. - shared_experts_input.record_stream(self._stream) - - # Mark sync start point for the aux stream since we will - # run in parallel with router/gate. - self._stream.wait_stream(current_stream()) - - def _run_in_aux_stream( - self, - shared_experts_input: torch.Tensor, - ) -> torch.Tensor: - # TODO: assert that maybe_sync_shared_experts_stream has been called. - - # Run shared experts in parallel on a separate stream. + Returns true if the shared experts were enqueued, false otherwise. Call + `wait` to wait for the shared experts to finish if this returns true. + """ + if ( + self._determine_shared_experts_order(shared_experts_input) + != SharedExpertsOrder.MULTI_STREAM_OVERLAPPED + ): + return False + assert self._stream is not None + idx = self._output_idx + assert self._output[idx] is None + self._input_ready_event[idx].record(current_stream()) with torch.cuda.stream(self._stream): - output = self._layer(shared_experts_input) - current_stream().wait_stream(self._stream) + self._input_ready_event[idx].wait(self._stream) + self._output[idx] = self._layer(shared_experts_input) + self._output_ready_event[idx].record(self._stream) + return True - return output + def wait(self) -> None: + """Block the main stream until `maybe_forward_async` output is ready.""" + assert self._stream is not None + self._output_ready_event[self._output_idx].wait(current_stream()) @property def _output_idx(self) -> int: @@ -174,11 +181,6 @@ class SharedExperts(torch.nn.Module): assert self._output[self._output_idx] is None - if order == SharedExpertsOrder.MULTI_STREAM_OVERLAPPED: - self._output[self._output_idx] = self._run_in_aux_stream( - shared_experts_input - ) - else: - self._output[self._output_idx] = self._layer(shared_experts_input) + self._output[self._output_idx] = self._layer(shared_experts_input) assert self._output[self._output_idx] is not None