[Perf][MoE] Eliminate staging from NCCL symmetric reduce-scatter (#49194)

Signed-off-by: samnordmann <snordmann@nvidia.com>
Co-authored-by: Misha Goin <mgoin64@gmail.com>
This commit is contained in:
Samuel Nordmann
2026-10-06 07:37:01 -07:00
committed by GitHub
co-authored by Misha Goin
parent bf049ddb8f
commit 4a99ca40d6
8 changed files with 723 additions and 69 deletions
@@ -197,6 +197,20 @@ class FusedMoEPrepareAndFinalize(ABC):
"""
return
def allocate_fused_expert_output(
self,
shape: tuple[int, ...],
dtype: torch.dtype,
device: torch.device,
weight_and_reduce_impl: TopKWeightAndReduce,
) -> torch.Tensor | None:
"""Optionally provide the exact output buffer for the expert kernel.
Implementations returning a tensor must preserve its identity through
finalize; replacing it may silently reintroduce a write-back copy.
"""
return None
@property
@abstractmethod
def activation_format(self) -> FusedMoEActivationFormat:
@@ -1054,13 +1068,14 @@ class FusedMoEKernelModularImpl:
local_num_experts: int,
expert_tokens_meta: ExpertTokensMetadata | None,
activation: MoEActivation,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, bool]:
"""Allocate temporary and output buffers for the fused experts op.
Inputs:
- out_dtype: output type of workspace and output tensors.
- device: the device of the workspace and output tensors.
See `workspace_shapes` for a description of the remainder of arguments.
Returns a tuple of (workspace13, workspace2, output) tensors.
Returns the workspace tensors, output tensor, and whether the output
uses a prepare/finalize-provided allocation.
"""
assert M_full > 0 and M_chunk > 0
@@ -1090,9 +1105,20 @@ class FusedMoEKernelModularImpl:
activation,
)
fused_out_alias = self.prepare_finalize.allocate_fused_expert_output(
fused_out_shape,
workspace_dtype,
device,
self.fused_experts.finalize_weight_and_reduce_impl(),
)
# We can reuse the memory between cache1 and cache3 because by the
# time we need cache3, we're done with cache1.
# Reuse workspace13 for the output since there is only one chunk.
# Keep the original fused-output reservation even when the output is
# externally aliased. The workspace manager profiles and locks a
# process-wide high-water mark; later graph shapes may need this
# storage for workspace13.
max_shape_size = max(prod(workspace13_shape), prod(fused_out_shape))
if current_platform.is_cpu():
@@ -1110,16 +1136,23 @@ class FusedMoEKernelModularImpl:
)
workspace13 = _resize_cache(common_workspace, workspace13_shape)
fused_out = _resize_cache(common_workspace, fused_out_shape)
return workspace13, workspace2, fused_out
return workspace13, workspace2, fused_out, False
common_workspace, workspace2 = current_workspace_manager().get_simultaneous(
((max_shape_size,), workspace_dtype),
(workspace2_shape, workspace_dtype),
)
workspace13 = _resize_cache(common_workspace, workspace13_shape)
fused_out = _resize_cache(common_workspace, fused_out_shape)
if fused_out_alias is None:
fused_out = _resize_cache(common_workspace, fused_out_shape)
else:
assert fused_out_alias.shape == fused_out_shape
assert fused_out_alias.dtype == workspace_dtype
assert fused_out_alias.device == device
assert fused_out_alias.is_contiguous()
fused_out = fused_out_alias
return workspace13, workspace2, fused_out
return workspace13, workspace2, fused_out, fused_out_alias is not None
def _maybe_apply_shared_experts(
self,
@@ -1251,22 +1284,25 @@ class FusedMoEKernelModularImpl:
if M_full == 0:
return torch.empty_like(a1q, dtype=in_dtype)
workspace13, workspace2, fused_out = self._allocate_buffers(
in_dtype,
a1q.device,
M_full,
M_full,
N,
K,
top_k,
global_num_experts,
local_num_experts,
expert_tokens_meta,
activation,
workspace13, workspace2, fused_out, fused_out_is_specialized = (
self._allocate_buffers(
in_dtype,
a1q.device,
M_full,
M_full,
N,
K,
top_k,
global_num_experts,
local_num_experts,
expert_tokens_meta,
activation,
)
)
use_output_alias = (
output_alias is not None
not fused_out_is_specialized
and output_alias is not None
and output_alias.shape == fused_out.shape
and output_alias.dtype == fused_out.dtype
and output_alias.device == fused_out.device
@@ -8,6 +8,7 @@ from vllm.model_executor.layers.fused_moe.config import FusedMoEQuantConfig
from vllm.model_executor.layers.fused_moe.topk_weight_and_reduce import (
TopKWeightAndReduceContiguous,
TopKWeightAndReduceDelegate,
TopKWeightAndReduceNoOP,
)
from vllm.model_executor.layers.fused_moe.utils import moe_kernel_quantize_input
from vllm.utils.flashinfer import nvfp4_block_scale_interleave
@@ -108,6 +109,30 @@ class MoEPrepareAndFinalizeNaiveDPEPModular(mk.FusedMoEPrepareAndFinalizeModular
def output_is_reduced(self) -> bool:
return False
def _allocate_combine_input(
self,
shape: tuple[int, ...],
dtype: torch.dtype,
device: torch.device,
) -> torch.Tensor | None:
return get_ep_group().allocate_combine_input(
shape,
dtype,
device,
is_sequence_parallel=self.is_sequence_parallel,
)
def allocate_fused_expert_output(
self,
shape: tuple[int, ...],
dtype: torch.dtype,
device: torch.device,
weight_and_reduce_impl: mk.TopKWeightAndReduce,
) -> torch.Tensor | None:
if isinstance(weight_and_reduce_impl, TopKWeightAndReduceNoOP):
return self._allocate_combine_input(shape, dtype, device)
return None
def prepare(
self,
a1: torch.Tensor,
@@ -194,16 +219,29 @@ class MoEPrepareAndFinalizeNaiveDPEPModular(mk.FusedMoEPrepareAndFinalizeModular
if isinstance(weight_and_reduce_impl, TopKWeightAndReduceDelegate):
weight_and_reduce_impl = TopKWeightAndReduceContiguous()
if isinstance(weight_and_reduce_impl, TopKWeightAndReduceNoOP):
# allocate_fused_expert_output already selected the combine input.
# Reuse it directly; the symmetric allocator may return a new view
# object for the same high-water backing allocation.
combine_input = fused_expert_output
else:
combine_input = self._allocate_combine_input(
(topk_ids.shape[0], fused_expert_output.shape[-1]),
fused_expert_output.dtype,
fused_expert_output.device,
)
out = weight_and_reduce_impl.apply(
output=None,
output=combine_input,
fused_expert_output=fused_expert_output,
topk_weights=topk_weights,
topk_ids=topk_ids,
apply_router_weight_on_input=apply_router_weight_on_input,
)
output.copy_(
get_ep_group().combine(out, is_sequence_parallel=self.is_sequence_parallel)
get_ep_group().combine_into_output(
out,
output,
is_sequence_parallel=self.is_sequence_parallel,
)