Add pynccl all-gatherv and reducescatterv (#20154)
Signed-off-by: Trevor Morris <tmorris@nvidia.com> Signed-off-by: mgoin <mgoin64@gmail.com> Co-authored-by: mgoin <mgoin64@gmail.com>
This commit is contained in:
@@ -383,6 +383,12 @@ class GroupCoordinator:
|
||||
dim: int) -> torch.Tensor:
|
||||
return self.device_communicator.all_gather(input_, dim)
|
||||
|
||||
def all_gatherv(self,
|
||||
input_: Union[torch.Tensor, list[torch.Tensor]],
|
||||
dim: int = 0,
|
||||
sizes: Optional[list[int]] = None):
|
||||
return self.device_communicator.all_gatherv(input_, dim, sizes)
|
||||
|
||||
def reduce_scatter(self,
|
||||
input_: torch.Tensor,
|
||||
dim: int = -1) -> torch.Tensor:
|
||||
@@ -401,6 +407,12 @@ class GroupCoordinator:
|
||||
else:
|
||||
return self._reduce_scatter_out_place(input_, dim)
|
||||
|
||||
def reduce_scatterv(self,
|
||||
input_: torch.Tensor,
|
||||
dim: int = -1,
|
||||
sizes: Optional[list[int]] = None) -> torch.Tensor:
|
||||
return self.device_communicator.reduce_scatterv(input_, dim, sizes)
|
||||
|
||||
def _reduce_scatter_out_place(self, input_: torch.Tensor,
|
||||
dim: int) -> torch.Tensor:
|
||||
return self.device_communicator.reduce_scatter(input_, dim)
|
||||
|
||||
Reference in New Issue
Block a user