[Feature] Pipeline Parallel Async send/recv, 2.9% E2E throughput improvement (#33368)

Signed-off-by: yewentao256 <zhyanwentao@126.com>
Signed-off-by: Wentao Ye <44945378+yewentao256@users.noreply.github.com>
Co-authored-by: Nick Hill <nhill@redhat.com>
This commit is contained in:
Wentao Ye
2026-02-13 16:38:16 +08:00
committed by GitHub
co-authored by Nick Hill
parent dddbff4624
commit 3d2a026fd0
3 changed files with 298 additions and 81 deletions
+137 -76
View File
@@ -33,7 +33,7 @@ from contextlib import contextmanager, nullcontext
from dataclasses import dataclass
from datetime import timedelta
from multiprocessing import shared_memory
from typing import Any
from typing import Any, Protocol
from unittest.mock import patch
import torch
@@ -64,6 +64,14 @@ class GraphCaptureContext:
TensorMetadata = namedtuple("TensorMetadata", ["device", "dtype", "size"])
class Handle(Protocol):
"""Minimal async work handle used by P2P send/recv methods."""
def is_completed(self) -> bool: ...
def wait(self) -> None: ...
def _split_tensor_dict(
tensor_dict: dict[str, torch.Tensor | Any],
) -> tuple[list[tuple[str, Any]], list[torch.Tensor]]:
@@ -780,6 +788,20 @@ class GroupCoordinator:
async_handle.wait()
return tensor_dict
def _should_use_all_gather(
self,
key: str,
numel: int,
all_gather_group: "GroupCoordinator | None",
all_gather_tensors: dict[str, bool] | None,
) -> bool:
if all_gather_group is None:
return False
use_all_gather = numel % all_gather_group.world_size == 0
if all_gather_tensors is not None:
use_all_gather = all_gather_tensors.get(key, use_all_gather)
return use_all_gather
def send_tensor_dict(
self,
tensor_dict: dict[str, torch.Tensor | Any],
@@ -808,6 +830,35 @@ class GroupCoordinator:
# Bypass the function if we are using only 1 GPU.
if not torch.distributed.is_initialized() or self.world_size == 1:
return tensor_dict
handles = self.isend_tensor_dict(
tensor_dict,
dst=dst,
all_gather_group=all_gather_group,
all_gather_tensors=all_gather_tensors,
)
for handle in handles:
handle.wait()
return None
def isend_tensor_dict(
self,
tensor_dict: dict[str, torch.Tensor | Any],
dst: int | None = None,
all_gather_group: "GroupCoordinator | None" = None,
all_gather_tensors: dict[str, bool] | None = None,
) -> list[Handle]:
if self.world_size <= 1:
return []
if self.use_cpu_custom_send_recv:
if self.device_communicator is None:
raise ValueError("No device communicator found")
# custom device communicator path is synchronous
self.device_communicator.send_tensor_dict( # type: ignore
tensor_dict, dst
)
return []
all_gather_size = 1 if all_gather_group is None else all_gather_group.world_size
all_gather_rank = (
0 if all_gather_group is None else all_gather_group.rank_in_group
@@ -820,53 +871,31 @@ class GroupCoordinator:
dst = (self.rank_in_group + 1) % self.world_size
assert dst < self.world_size, f"Invalid dst rank ({dst})"
if self.use_cpu_custom_send_recv:
if self.device_communicator is None:
raise ValueError("No device communicator found")
self.device_communicator.send_tensor_dict( # type: ignore
tensor_dict, dst
)
return None
metadata_list: list[tuple[Any, Any]] = []
assert isinstance(tensor_dict, dict), (
f"Expecting a dictionary, got {type(tensor_dict)}"
)
metadata_list, tensor_list = _split_tensor_dict(tensor_dict)
# `metadata_list` lives in CPU memory.
# `send_object_list` has serialization & deserialization,
# all happening on CPU. Therefore, we can use the CPU group.
self.send_object(metadata_list, dst=dst)
tensor_keys = [k for k, v in tensor_dict.items() if isinstance(v, torch.Tensor)]
assert len(tensor_keys) == len(tensor_list)
handles: list[Handle] = []
for key, tensor in zip(tensor_keys, tensor_list):
if tensor.numel() == 0:
# Skip sending empty tensors.
continue
# send-allgather: send only a slice, then do allgather.
use_all_gather = (
all_gather_group is not None and tensor.numel() % all_gather_size == 0
)
use_all_gather = (
all_gather_tensors.get(key, use_all_gather)
if all_gather_tensors
else use_all_gather
)
if use_all_gather:
if self._should_use_all_gather(
key, tensor.numel(), all_gather_group, all_gather_tensors
):
tensor = tensor.reshape(all_gather_size, -1)[all_gather_rank]
if tensor.is_cpu:
# use metadata_group for CPU tensors
torch.distributed.send(
tensor, dst=self.ranks[dst], group=metadata_group
)
else:
# use group for GPU tensors
torch.distributed.send(tensor, dst=self.ranks[dst], group=group)
return None
comm_group = metadata_group if tensor.is_cpu else group
handle = torch.distributed.isend(
tensor, dst=self.ranks[dst], group=comm_group
)
if tensor.is_cuda:
tensor.record_stream(torch.cuda.current_stream(tensor.device))
handles.append(handle)
return handles
def recv_tensor_dict(
self,
@@ -895,6 +924,38 @@ class GroupCoordinator:
# Bypass the function if we are using only 1 GPU.
if not torch.distributed.is_initialized() or self.world_size == 1:
return None
tensor_dict, handles, postprocess = self.irecv_tensor_dict(
src=src,
all_gather_group=all_gather_group,
all_gather_tensors=all_gather_tensors,
)
for handle in handles:
handle.wait()
for fn in postprocess:
fn()
return tensor_dict
def irecv_tensor_dict(
self,
src: int | None = None,
all_gather_group: "GroupCoordinator | None" = None,
all_gather_tensors: dict[str, bool] | None = None,
) -> tuple[
dict[str, torch.Tensor | Any] | None,
list[Handle],
list[Callable[[], None]],
]:
if not torch.distributed.is_initialized() or self.world_size == 1:
return None, [], []
if self.use_cpu_custom_send_recv:
if self.device_communicator is None:
raise ValueError("No device communicator found")
# custom device communicator path is synchronous
sync_tensor_dict = self.device_communicator.recv_tensor_dict( # type: ignore
src
)
return sync_tensor_dict, [], []
all_gather_size = 1 if all_gather_group is None else all_gather_group.world_size
all_gather_rank = (
0 if all_gather_group is None else all_gather_group.rank_in_group
@@ -907,57 +968,57 @@ class GroupCoordinator:
src = (self.rank_in_group - 1) % self.world_size
assert src < self.world_size, f"Invalid src rank ({src})"
if self.use_cpu_custom_send_recv:
if self.device_communicator is None:
raise ValueError("No device communicator found")
return self.device_communicator.recv_tensor_dict( # type: ignore
src
)
recv_metadata_list = self.recv_object(src=src)
tensor_dict: dict[str, Any] = {}
handles: list[Handle] = []
postprocess: list[Callable[[], None]] = []
for key, value in recv_metadata_list:
if isinstance(value, TensorMetadata):
tensor = torch.empty(value.size, dtype=value.dtype, device=value.device)
if tensor.numel() == 0:
# Skip broadcasting empty tensors.
tensor_dict[key] = tensor
full_tensor = torch.empty(
value.size, dtype=value.dtype, device=value.device
)
if full_tensor.numel() == 0:
tensor_dict[key] = full_tensor
continue
# send-allgather: send only a slice, then do allgather.
use_all_gather = (
all_gather_group is not None
and tensor.numel() % all_gather_size == 0
)
use_all_gather = (
all_gather_tensors.get(key, use_all_gather)
if all_gather_tensors
else use_all_gather
)
if use_all_gather:
orig_shape = tensor.shape
tensor = tensor.reshape(all_gather_size, -1)[all_gather_rank]
if tensor.is_cpu:
# use metadata_group for CPU tensors
torch.distributed.recv(
tensor, src=self.ranks[src], group=metadata_group
if self._should_use_all_gather(
key, full_tensor.numel(), all_gather_group, all_gather_tensors
):
orig_shape = full_tensor.shape
slice_tensor = full_tensor.reshape(all_gather_size, -1)[
all_gather_rank
]
comm_group = metadata_group if slice_tensor.is_cpu else group
handle = torch.distributed.irecv(
slice_tensor, src=self.ranks[src], group=comm_group
)
handles.append(handle)
def _postprocess(
key: str = key,
slice_tensor: torch.Tensor = slice_tensor,
orig_shape: tuple[int, ...] = tuple(orig_shape),
all_gather_group=all_gather_group,
) -> None:
assert all_gather_group is not None
tensor_dict[key] = all_gather_group.all_gather(
slice_tensor, dim=0
).reshape(orig_shape)
postprocess.append(_postprocess)
tensor_dict[key] = slice_tensor
else:
# use group for GPU tensors
torch.distributed.recv(tensor, src=self.ranks[src], group=group)
if use_all_gather:
# do the allgather
tensor = all_gather_group.all_gather( # type: ignore
tensor, dim=0
comm_group = metadata_group if full_tensor.is_cpu else group
handle = torch.distributed.irecv(
full_tensor, src=self.ranks[src], group=comm_group
)
tensor = tensor.reshape(orig_shape)
tensor_dict[key] = tensor
handles.append(handle)
tensor_dict[key] = full_tensor
else:
tensor_dict[key] = value
return tensor_dict
return tensor_dict, handles, postprocess
def barrier(self):
"""Barrier synchronization among the group.