[Perf][MoE] Eliminate staging from NCCL symmetric reduce-scatter (#49194)

Signed-off-by: samnordmann <snordmann@nvidia.com>
Co-authored-by: Misha Goin <mgoin64@gmail.com>
This commit is contained in:
Samuel Nordmann
2026-10-06 07:37:01 -07:00
committed by GitHub
co-authored by Misha Goin
parent bf049ddb8f
commit 4a99ca40d6
8 changed files with 723 additions and 69 deletions
+39
View File
@@ -801,6 +801,19 @@ class GroupCoordinator:
raise ValueError("No device communicator found")
return self.device_communicator.reduce_scatterv(input_, dim, sizes)
def reduce_scatterv_into_output(
self,
input_: torch.Tensor,
output: torch.Tensor,
dim: int = -1,
sizes: list[int] | None = None,
) -> torch.Tensor:
if self.device_communicator is None:
raise ValueError("No device communicator found")
return self.device_communicator.reduce_scatterv_into_output(
input_, output, dim, sizes
)
def _reduce_scatter_out_place(self, input_: torch.Tensor, dim: int) -> torch.Tensor:
if self.device_communicator is None:
raise ValueError("No device communicator found")
@@ -1442,6 +1455,32 @@ class GroupCoordinator:
else:
return hidden_states
def allocate_combine_input(
self,
shape: tuple[int, ...],
dtype: torch.dtype,
device: torch.device,
is_sequence_parallel: bool = False,
) -> torch.Tensor | None:
if self.device_communicator is None:
return None
return self.device_communicator.allocate_combine_input(
shape, dtype, device, is_sequence_parallel
)
def combine_into_output(
self,
hidden_states: torch.Tensor,
output: torch.Tensor,
is_sequence_parallel: bool = False,
) -> torch.Tensor:
if self.device_communicator is None:
output.copy_(hidden_states)
return output
return self.device_communicator.combine_into_output(
hidden_states, output, is_sequence_parallel
)
_WORLD: GroupCoordinator | None = None
_INNER_DP_WORLD: GroupCoordinator | None = None