[Perf][MoE] Eliminate staging from NCCL symmetric reduce-scatter (#49194)

Signed-off-by: samnordmann <snordmann@nvidia.com>
Co-authored-by: Misha Goin <mgoin64@gmail.com>
This commit is contained in:
Samuel Nordmann
2026-10-06 07:37:01 -07:00
committed by GitHub
co-authored by Misha Goin
parent bf049ddb8f
commit 4a99ca40d6
8 changed files with 723 additions and 69 deletions
@@ -171,6 +171,42 @@ class AgRsAll2AllManager(All2AllManagerBase):
hidden_states = dist_group.reduce_scatterv(hidden_states, dim=0, sizes=sizes)
return hidden_states
def allocate_combine_input(
self,
shape: tuple[int, ...],
dtype: torch.dtype,
device: torch.device,
is_sequence_parallel: bool = False,
) -> torch.Tensor | None:
dist_group = self._get_comm_group(is_sequence_parallel)
sizes = self._get_sizes(shape[0] // dist_group.world_size, dist_group)
if sum(sizes) != shape[0] or any(size != sizes[0] for size in sizes):
return None
device_communicator = dist_group.device_communicator
if device_communicator is None:
return None
return device_communicator.get_symmetric_memory_buffer(
"moe_ag_rs_combine", shape, dtype, device
)
def combine_into_output(
self,
hidden_states: torch.Tensor,
output: torch.Tensor,
is_sequence_parallel: bool = False,
) -> torch.Tensor:
dist_group = self._get_comm_group(is_sequence_parallel)
sizes = self._get_sizes(
hidden_states.shape[0] // dist_group.world_size,
dist_group,
)
return dist_group.reduce_scatterv_into_output(
hidden_states,
output,
dim=0,
sizes=sizes,
)
def destroy(self):
pass