[Feature] Pipeline Parallel Async send/recv, 2.9% E2E throughput improvement (#33368)
Signed-off-by: yewentao256 <zhyanwentao@126.com> Signed-off-by: Wentao Ye <44945378+yewentao256@users.noreply.github.com> Co-authored-by: Nick Hill <nhill@redhat.com>
This commit is contained in:
@@ -33,7 +33,7 @@ from contextlib import contextmanager, nullcontext
|
||||
from dataclasses import dataclass
|
||||
from datetime import timedelta
|
||||
from multiprocessing import shared_memory
|
||||
from typing import Any
|
||||
from typing import Any, Protocol
|
||||
from unittest.mock import patch
|
||||
|
||||
import torch
|
||||
@@ -64,6 +64,14 @@ class GraphCaptureContext:
|
||||
TensorMetadata = namedtuple("TensorMetadata", ["device", "dtype", "size"])
|
||||
|
||||
|
||||
class Handle(Protocol):
|
||||
"""Minimal async work handle used by P2P send/recv methods."""
|
||||
|
||||
def is_completed(self) -> bool: ...
|
||||
|
||||
def wait(self) -> None: ...
|
||||
|
||||
|
||||
def _split_tensor_dict(
|
||||
tensor_dict: dict[str, torch.Tensor | Any],
|
||||
) -> tuple[list[tuple[str, Any]], list[torch.Tensor]]:
|
||||
@@ -780,6 +788,20 @@ class GroupCoordinator:
|
||||
async_handle.wait()
|
||||
return tensor_dict
|
||||
|
||||
def _should_use_all_gather(
|
||||
self,
|
||||
key: str,
|
||||
numel: int,
|
||||
all_gather_group: "GroupCoordinator | None",
|
||||
all_gather_tensors: dict[str, bool] | None,
|
||||
) -> bool:
|
||||
if all_gather_group is None:
|
||||
return False
|
||||
use_all_gather = numel % all_gather_group.world_size == 0
|
||||
if all_gather_tensors is not None:
|
||||
use_all_gather = all_gather_tensors.get(key, use_all_gather)
|
||||
return use_all_gather
|
||||
|
||||
def send_tensor_dict(
|
||||
self,
|
||||
tensor_dict: dict[str, torch.Tensor | Any],
|
||||
@@ -808,6 +830,35 @@ class GroupCoordinator:
|
||||
# Bypass the function if we are using only 1 GPU.
|
||||
if not torch.distributed.is_initialized() or self.world_size == 1:
|
||||
return tensor_dict
|
||||
handles = self.isend_tensor_dict(
|
||||
tensor_dict,
|
||||
dst=dst,
|
||||
all_gather_group=all_gather_group,
|
||||
all_gather_tensors=all_gather_tensors,
|
||||
)
|
||||
for handle in handles:
|
||||
handle.wait()
|
||||
return None
|
||||
|
||||
def isend_tensor_dict(
|
||||
self,
|
||||
tensor_dict: dict[str, torch.Tensor | Any],
|
||||
dst: int | None = None,
|
||||
all_gather_group: "GroupCoordinator | None" = None,
|
||||
all_gather_tensors: dict[str, bool] | None = None,
|
||||
) -> list[Handle]:
|
||||
if self.world_size <= 1:
|
||||
return []
|
||||
|
||||
if self.use_cpu_custom_send_recv:
|
||||
if self.device_communicator is None:
|
||||
raise ValueError("No device communicator found")
|
||||
# custom device communicator path is synchronous
|
||||
self.device_communicator.send_tensor_dict( # type: ignore
|
||||
tensor_dict, dst
|
||||
)
|
||||
return []
|
||||
|
||||
all_gather_size = 1 if all_gather_group is None else all_gather_group.world_size
|
||||
all_gather_rank = (
|
||||
0 if all_gather_group is None else all_gather_group.rank_in_group
|
||||
@@ -820,53 +871,31 @@ class GroupCoordinator:
|
||||
dst = (self.rank_in_group + 1) % self.world_size
|
||||
assert dst < self.world_size, f"Invalid dst rank ({dst})"
|
||||
|
||||
if self.use_cpu_custom_send_recv:
|
||||
if self.device_communicator is None:
|
||||
raise ValueError("No device communicator found")
|
||||
self.device_communicator.send_tensor_dict( # type: ignore
|
||||
tensor_dict, dst
|
||||
)
|
||||
return None
|
||||
|
||||
metadata_list: list[tuple[Any, Any]] = []
|
||||
assert isinstance(tensor_dict, dict), (
|
||||
f"Expecting a dictionary, got {type(tensor_dict)}"
|
||||
)
|
||||
metadata_list, tensor_list = _split_tensor_dict(tensor_dict)
|
||||
# `metadata_list` lives in CPU memory.
|
||||
# `send_object_list` has serialization & deserialization,
|
||||
# all happening on CPU. Therefore, we can use the CPU group.
|
||||
self.send_object(metadata_list, dst=dst)
|
||||
|
||||
tensor_keys = [k for k, v in tensor_dict.items() if isinstance(v, torch.Tensor)]
|
||||
assert len(tensor_keys) == len(tensor_list)
|
||||
|
||||
handles: list[Handle] = []
|
||||
for key, tensor in zip(tensor_keys, tensor_list):
|
||||
if tensor.numel() == 0:
|
||||
# Skip sending empty tensors.
|
||||
continue
|
||||
|
||||
# send-allgather: send only a slice, then do allgather.
|
||||
use_all_gather = (
|
||||
all_gather_group is not None and tensor.numel() % all_gather_size == 0
|
||||
)
|
||||
use_all_gather = (
|
||||
all_gather_tensors.get(key, use_all_gather)
|
||||
if all_gather_tensors
|
||||
else use_all_gather
|
||||
)
|
||||
if use_all_gather:
|
||||
if self._should_use_all_gather(
|
||||
key, tensor.numel(), all_gather_group, all_gather_tensors
|
||||
):
|
||||
tensor = tensor.reshape(all_gather_size, -1)[all_gather_rank]
|
||||
|
||||
if tensor.is_cpu:
|
||||
# use metadata_group for CPU tensors
|
||||
torch.distributed.send(
|
||||
tensor, dst=self.ranks[dst], group=metadata_group
|
||||
)
|
||||
else:
|
||||
# use group for GPU tensors
|
||||
torch.distributed.send(tensor, dst=self.ranks[dst], group=group)
|
||||
return None
|
||||
comm_group = metadata_group if tensor.is_cpu else group
|
||||
handle = torch.distributed.isend(
|
||||
tensor, dst=self.ranks[dst], group=comm_group
|
||||
)
|
||||
if tensor.is_cuda:
|
||||
tensor.record_stream(torch.cuda.current_stream(tensor.device))
|
||||
handles.append(handle)
|
||||
|
||||
return handles
|
||||
|
||||
def recv_tensor_dict(
|
||||
self,
|
||||
@@ -895,6 +924,38 @@ class GroupCoordinator:
|
||||
# Bypass the function if we are using only 1 GPU.
|
||||
if not torch.distributed.is_initialized() or self.world_size == 1:
|
||||
return None
|
||||
tensor_dict, handles, postprocess = self.irecv_tensor_dict(
|
||||
src=src,
|
||||
all_gather_group=all_gather_group,
|
||||
all_gather_tensors=all_gather_tensors,
|
||||
)
|
||||
for handle in handles:
|
||||
handle.wait()
|
||||
for fn in postprocess:
|
||||
fn()
|
||||
return tensor_dict
|
||||
|
||||
def irecv_tensor_dict(
|
||||
self,
|
||||
src: int | None = None,
|
||||
all_gather_group: "GroupCoordinator | None" = None,
|
||||
all_gather_tensors: dict[str, bool] | None = None,
|
||||
) -> tuple[
|
||||
dict[str, torch.Tensor | Any] | None,
|
||||
list[Handle],
|
||||
list[Callable[[], None]],
|
||||
]:
|
||||
if not torch.distributed.is_initialized() or self.world_size == 1:
|
||||
return None, [], []
|
||||
if self.use_cpu_custom_send_recv:
|
||||
if self.device_communicator is None:
|
||||
raise ValueError("No device communicator found")
|
||||
# custom device communicator path is synchronous
|
||||
sync_tensor_dict = self.device_communicator.recv_tensor_dict( # type: ignore
|
||||
src
|
||||
)
|
||||
return sync_tensor_dict, [], []
|
||||
|
||||
all_gather_size = 1 if all_gather_group is None else all_gather_group.world_size
|
||||
all_gather_rank = (
|
||||
0 if all_gather_group is None else all_gather_group.rank_in_group
|
||||
@@ -907,57 +968,57 @@ class GroupCoordinator:
|
||||
src = (self.rank_in_group - 1) % self.world_size
|
||||
assert src < self.world_size, f"Invalid src rank ({src})"
|
||||
|
||||
if self.use_cpu_custom_send_recv:
|
||||
if self.device_communicator is None:
|
||||
raise ValueError("No device communicator found")
|
||||
return self.device_communicator.recv_tensor_dict( # type: ignore
|
||||
src
|
||||
)
|
||||
|
||||
recv_metadata_list = self.recv_object(src=src)
|
||||
tensor_dict: dict[str, Any] = {}
|
||||
handles: list[Handle] = []
|
||||
postprocess: list[Callable[[], None]] = []
|
||||
|
||||
for key, value in recv_metadata_list:
|
||||
if isinstance(value, TensorMetadata):
|
||||
tensor = torch.empty(value.size, dtype=value.dtype, device=value.device)
|
||||
if tensor.numel() == 0:
|
||||
# Skip broadcasting empty tensors.
|
||||
tensor_dict[key] = tensor
|
||||
full_tensor = torch.empty(
|
||||
value.size, dtype=value.dtype, device=value.device
|
||||
)
|
||||
if full_tensor.numel() == 0:
|
||||
tensor_dict[key] = full_tensor
|
||||
continue
|
||||
|
||||
# send-allgather: send only a slice, then do allgather.
|
||||
use_all_gather = (
|
||||
all_gather_group is not None
|
||||
and tensor.numel() % all_gather_size == 0
|
||||
)
|
||||
use_all_gather = (
|
||||
all_gather_tensors.get(key, use_all_gather)
|
||||
if all_gather_tensors
|
||||
else use_all_gather
|
||||
)
|
||||
|
||||
if use_all_gather:
|
||||
orig_shape = tensor.shape
|
||||
tensor = tensor.reshape(all_gather_size, -1)[all_gather_rank]
|
||||
|
||||
if tensor.is_cpu:
|
||||
# use metadata_group for CPU tensors
|
||||
torch.distributed.recv(
|
||||
tensor, src=self.ranks[src], group=metadata_group
|
||||
if self._should_use_all_gather(
|
||||
key, full_tensor.numel(), all_gather_group, all_gather_tensors
|
||||
):
|
||||
orig_shape = full_tensor.shape
|
||||
slice_tensor = full_tensor.reshape(all_gather_size, -1)[
|
||||
all_gather_rank
|
||||
]
|
||||
comm_group = metadata_group if slice_tensor.is_cpu else group
|
||||
handle = torch.distributed.irecv(
|
||||
slice_tensor, src=self.ranks[src], group=comm_group
|
||||
)
|
||||
handles.append(handle)
|
||||
|
||||
def _postprocess(
|
||||
key: str = key,
|
||||
slice_tensor: torch.Tensor = slice_tensor,
|
||||
orig_shape: tuple[int, ...] = tuple(orig_shape),
|
||||
all_gather_group=all_gather_group,
|
||||
) -> None:
|
||||
assert all_gather_group is not None
|
||||
tensor_dict[key] = all_gather_group.all_gather(
|
||||
slice_tensor, dim=0
|
||||
).reshape(orig_shape)
|
||||
|
||||
postprocess.append(_postprocess)
|
||||
tensor_dict[key] = slice_tensor
|
||||
else:
|
||||
# use group for GPU tensors
|
||||
torch.distributed.recv(tensor, src=self.ranks[src], group=group)
|
||||
if use_all_gather:
|
||||
# do the allgather
|
||||
tensor = all_gather_group.all_gather( # type: ignore
|
||||
tensor, dim=0
|
||||
comm_group = metadata_group if full_tensor.is_cpu else group
|
||||
handle = torch.distributed.irecv(
|
||||
full_tensor, src=self.ranks[src], group=comm_group
|
||||
)
|
||||
tensor = tensor.reshape(orig_shape)
|
||||
|
||||
tensor_dict[key] = tensor
|
||||
handles.append(handle)
|
||||
tensor_dict[key] = full_tensor
|
||||
else:
|
||||
tensor_dict[key] = value
|
||||
return tensor_dict
|
||||
|
||||
return tensor_dict, handles, postprocess
|
||||
|
||||
def barrier(self):
|
||||
"""Barrier synchronization among the group.
|
||||
|
||||
Reference in New Issue
Block a user