[Bugfix] Fix sequence parallelism bug when enable pipeline parallelism (#24021)
Signed-off-by: cascade812 <cascade812@outlook.com>
This commit is contained in:
@@ -663,14 +663,29 @@ class GroupCoordinator:
|
||||
tensor_dict: dict[str, Union[torch.Tensor, Any]],
|
||||
dst: Optional[int] = None,
|
||||
all_gather_group: Optional["GroupCoordinator"] = None,
|
||||
all_gather_tensors: Optional[dict[str, bool]] = None,
|
||||
) -> Optional[dict[str, Union[torch.Tensor, Any]]]:
|
||||
"""Send the input tensor dictionary.
|
||||
NOTE: `dst` is the local rank of the source rank.
|
||||
|
||||
all_gather_group: The group for the all-gather operation. If provided,
|
||||
an optimization is enabled where each rank in the group sends a
|
||||
slice of a tensor and the receiver reconstructs it using an
|
||||
all-gather, which can improve performance. This is typically the
|
||||
tensor-parallel group.
|
||||
all_gather_tensors: A dictionary to specify which tensors should use
|
||||
the all-gather optimization, which is only effective when
|
||||
`all_gather_group` is provided. By default, this optimization is
|
||||
on for any tensor whose size is divisible by the
|
||||
`all_gather_group`'s world size. However, it should be disabled
|
||||
for tensors that are not fully replicated across the group (e.g.,
|
||||
the residual tensor when sequence parallelism is enabled). This
|
||||
dictionary allows overriding the default behavior on a per-tensor
|
||||
basis.
|
||||
"""
|
||||
# Bypass the function if we are using only 1 GPU.
|
||||
if not torch.distributed.is_initialized() or self.world_size == 1:
|
||||
return tensor_dict
|
||||
|
||||
all_gather_size = (1 if all_gather_group is None else
|
||||
all_gather_group.world_size)
|
||||
all_gather_rank = (0 if all_gather_group is None else
|
||||
@@ -699,14 +714,23 @@ class GroupCoordinator:
|
||||
# `send_object_list` has serialization & deserialization,
|
||||
# all happening on CPU. Therefore, we can use the CPU group.
|
||||
self.send_object(metadata_list, dst=dst)
|
||||
for tensor in tensor_list:
|
||||
|
||||
tensor_keys = [
|
||||
k for k, v in tensor_dict.items() if isinstance(v, torch.Tensor)
|
||||
]
|
||||
assert len(tensor_keys) == len(tensor_list)
|
||||
|
||||
for key, tensor in zip(tensor_keys, tensor_list):
|
||||
if tensor.numel() == 0:
|
||||
# Skip sending empty tensors.
|
||||
continue
|
||||
|
||||
# send-allgather: send only a slice, then do allgather.
|
||||
if (all_gather_group is not None
|
||||
and tensor.numel() % all_gather_size == 0):
|
||||
use_all_gather = (all_gather_group is not None
|
||||
and tensor.numel() % all_gather_size == 0)
|
||||
use_all_gather = all_gather_tensors.get(key, use_all_gather) \
|
||||
if all_gather_tensors else use_all_gather
|
||||
if use_all_gather:
|
||||
tensor = tensor.reshape(all_gather_size, -1)[all_gather_rank]
|
||||
|
||||
if tensor.is_cpu:
|
||||
@@ -725,14 +749,29 @@ class GroupCoordinator:
|
||||
self,
|
||||
src: Optional[int] = None,
|
||||
all_gather_group: Optional["GroupCoordinator"] = None,
|
||||
all_gather_tensors: Optional[dict[str, bool]] = None,
|
||||
) -> Optional[dict[str, Union[torch.Tensor, Any]]]:
|
||||
"""Recv the input tensor dictionary.
|
||||
NOTE: `src` is the local rank of the source rank.
|
||||
|
||||
all_gather_group: The group for the all-gather operation. If provided,
|
||||
an optimization is enabled where each rank in the group sends a
|
||||
slice of a tensor and the receiver reconstructs it using an
|
||||
all-gather, which can improve performance. This is typically the
|
||||
tensor-parallel group.
|
||||
all_gather_tensors: A dictionary to specify which tensors should use
|
||||
the all-gather optimization, which is only effective when
|
||||
`all_gather_group` is provided. By default, this optimization is
|
||||
on for any tensor whose size is divisible by the
|
||||
`all_gather_group`'s world size. However, it should be disabled
|
||||
for tensors that are not fully replicated across the group (e.g.,
|
||||
the residual tensor when sequence parallelism is enabled). This
|
||||
dictionary allows overriding the default behavior on a per-tensor
|
||||
basis.
|
||||
"""
|
||||
# Bypass the function if we are using only 1 GPU.
|
||||
if not torch.distributed.is_initialized() or self.world_size == 1:
|
||||
return None
|
||||
|
||||
all_gather_size = (1 if all_gather_group is None else
|
||||
all_gather_group.world_size)
|
||||
all_gather_rank = (0 if all_gather_group is None else
|
||||
@@ -766,6 +805,8 @@ class GroupCoordinator:
|
||||
# send-allgather: send only a slice, then do allgather.
|
||||
use_all_gather = (all_gather_group is not None
|
||||
and tensor.numel() % all_gather_size == 0)
|
||||
use_all_gather = all_gather_tensors.get(key, use_all_gather) \
|
||||
if all_gather_tensors else use_all_gather
|
||||
|
||||
if use_all_gather:
|
||||
orig_shape = tensor.shape
|
||||
|
||||
Reference in New Issue
Block a user