[Distributed] Add send and recv helpers (#5719)
This commit is contained in:
@@ -20,6 +20,7 @@ If you only need to use the distributed environment without model/pipeline
|
||||
steps.
|
||||
"""
|
||||
import contextlib
|
||||
import pickle
|
||||
from collections import namedtuple
|
||||
from contextlib import contextmanager, nullcontext
|
||||
from dataclasses import dataclass
|
||||
@@ -28,6 +29,7 @@ from typing import Any, Dict, List, Optional, Tuple, Union
|
||||
from unittest.mock import patch
|
||||
|
||||
import torch
|
||||
import torch.distributed
|
||||
from torch.distributed import Backend, ProcessGroup
|
||||
|
||||
import vllm.envs as envs
|
||||
@@ -180,6 +182,16 @@ class GroupCoordinator:
|
||||
"""Return the global rank of the last process in the group"""
|
||||
return self.ranks[-1]
|
||||
|
||||
@property
|
||||
def is_first_rank(self):
|
||||
"""Return whether the caller is the first process in the group"""
|
||||
return self.rank == self.first_rank
|
||||
|
||||
@property
|
||||
def is_last_rank(self):
|
||||
"""Return whether the caller is the last process in the group"""
|
||||
return self.rank == self.last_rank
|
||||
|
||||
@property
|
||||
def next_rank(self):
|
||||
"""Return the global rank of the process that follows the caller"""
|
||||
@@ -374,6 +386,70 @@ class GroupCoordinator:
|
||||
group=self.device_group)
|
||||
return obj_list
|
||||
|
||||
def send_object(self, obj: Any, dst: int) -> None:
|
||||
"""Send the input object list to the destination rank."""
|
||||
"""NOTE: `dst` is the local rank of the destination rank."""
|
||||
|
||||
assert dst < self.world_size, f"Invalid dst rank ({dst})"
|
||||
|
||||
assert dst != self.rank, (
|
||||
"Invalid destination rank. Destination rank is the same "
|
||||
"as the current rank.")
|
||||
|
||||
# Serialize object to tensor and get the size as well
|
||||
object_tensor = torch.frombuffer(pickle.dumps(obj), dtype=torch.uint8)
|
||||
|
||||
size_tensor = torch.tensor([object_tensor.numel()],
|
||||
dtype=torch.long,
|
||||
device="cpu")
|
||||
|
||||
# Send object size
|
||||
|
||||
torch.distributed.send(size_tensor,
|
||||
dst=self.ranks[dst],
|
||||
group=self.cpu_group)
|
||||
|
||||
# Send object
|
||||
torch.distributed.send(object_tensor,
|
||||
dst=self.ranks[dst],
|
||||
group=self.cpu_group)
|
||||
|
||||
return None
|
||||
|
||||
def recv_object(self, src: int) -> Any:
|
||||
"""Receive the input object list from the source rank."""
|
||||
"""NOTE: `src` is the local rank of the source rank."""
|
||||
|
||||
assert src < self.world_size, f"Invalid src rank ({src})"
|
||||
|
||||
assert src != self.rank, (
|
||||
"Invalid source rank. Source rank is the same as the current rank."
|
||||
)
|
||||
|
||||
size_tensor = torch.empty(1, dtype=torch.long, device="cpu")
|
||||
|
||||
# Receive object size
|
||||
rank_size = torch.distributed.recv(size_tensor,
|
||||
src=src,
|
||||
group=self.cpu_group)
|
||||
|
||||
# Tensor to receive serialized objects into.
|
||||
object_tensor = torch.empty( # type: ignore[call-overload]
|
||||
size_tensor.item(), # type: ignore[arg-type]
|
||||
dtype=torch.uint8,
|
||||
device="cpu")
|
||||
|
||||
rank_object = torch.distributed.recv(object_tensor,
|
||||
src=src,
|
||||
group=self.cpu_group)
|
||||
|
||||
assert rank_object == rank_size, (
|
||||
"Received object sender rank does not match the size sender rank.")
|
||||
|
||||
obj = pickle.loads(object_tensor.numpy().tobytes())
|
||||
|
||||
return obj
|
||||
|
||||
def broadcast_tensor_dict(
|
||||
self,
|
||||
tensor_dict: Optional[Dict[Any, Union[torch.Tensor, Any]]] = None,
|
||||
@@ -459,6 +535,88 @@ class GroupCoordinator:
|
||||
async_handle.wait()
|
||||
return tensor_dict
|
||||
|
||||
def send_tensor_dict(
|
||||
self,
|
||||
tensor_dict: Dict[Any, Union[torch.Tensor, Any]],
|
||||
dst: Optional[int] = None
|
||||
) -> Optional[Dict[Any, Union[torch.Tensor, Any]]]:
|
||||
"""Send the input tensor dictionary.
|
||||
NOTE: `dst` is the local rank of the source rank.
|
||||
"""
|
||||
# Bypass the function if we are using only 1 GPU.
|
||||
if not torch.distributed.is_initialized() or self.world_size == 1:
|
||||
return tensor_dict
|
||||
|
||||
group = self.device_group
|
||||
metadata_group = self.cpu_group
|
||||
|
||||
if dst is None:
|
||||
dst = self.next_rank
|
||||
assert dst < self.world_size, f"Invalid dst rank ({dst})"
|
||||
|
||||
metadata_list: List[Tuple[Any, Any]] = []
|
||||
assert isinstance(
|
||||
tensor_dict,
|
||||
dict), f"Expecting a dictionary, got {type(tensor_dict)}"
|
||||
metadata_list, tensor_list = _split_tensor_dict(tensor_dict)
|
||||
# `metadata_list` lives in CPU memory.
|
||||
# `send_object_list` has serialization & deserialization,
|
||||
# all happening on CPU. Therefore, we can use the CPU group.
|
||||
self.send_object(metadata_list, dst=dst)
|
||||
for tensor in tensor_list:
|
||||
if tensor.numel() == 0:
|
||||
# Skip sending empty tensors.
|
||||
continue
|
||||
if tensor.is_cpu:
|
||||
# use metadata_group for CPU tensors
|
||||
torch.distributed.send(tensor, dst=dst, group=metadata_group)
|
||||
else:
|
||||
# use group for GPU tensors
|
||||
torch.distributed.send(tensor, dst=dst, group=group)
|
||||
return None
|
||||
|
||||
def recv_tensor_dict(
|
||||
self,
|
||||
src: Optional[int] = None
|
||||
) -> Optional[Dict[Any, Union[torch.Tensor, Any]]]:
|
||||
"""Recv the input tensor dictionary.
|
||||
NOTE: `src` is the local rank of the source rank.
|
||||
"""
|
||||
# Bypass the function if we are using only 1 GPU.
|
||||
if not torch.distributed.is_initialized() or self.world_size == 1:
|
||||
return None
|
||||
|
||||
group = self.device_group
|
||||
metadata_group = self.cpu_group
|
||||
|
||||
if src is None:
|
||||
src = self.prev_rank
|
||||
assert src < self.world_size, f"Invalid src rank ({src})"
|
||||
|
||||
recv_metadata_list = self.recv_object(src=src)
|
||||
tensor_dict = {}
|
||||
for key, value in recv_metadata_list:
|
||||
if isinstance(value, TensorMetadata):
|
||||
tensor = torch.empty(value.size,
|
||||
dtype=value.dtype,
|
||||
device=value.device)
|
||||
if tensor.numel() == 0:
|
||||
# Skip broadcasting empty tensors.
|
||||
tensor_dict[key] = tensor
|
||||
continue
|
||||
if tensor.is_cpu:
|
||||
# use metadata_group for CPU tensors
|
||||
torch.distributed.recv(tensor,
|
||||
src=src,
|
||||
group=metadata_group)
|
||||
else:
|
||||
# use group for GPU tensors
|
||||
torch.distributed.recv(tensor, src=src, group=group)
|
||||
tensor_dict[key] = tensor
|
||||
else:
|
||||
tensor_dict[key] = value
|
||||
return tensor_dict
|
||||
|
||||
def barrier(self):
|
||||
"""Barrier synchronization among the group.
|
||||
NOTE: don't use `device_group` here! `barrier` in NCCL is
|
||||
@@ -468,6 +626,35 @@ class GroupCoordinator:
|
||||
"""
|
||||
torch.distributed.barrier(group=self.cpu_group)
|
||||
|
||||
def send(self, tensor: torch.Tensor, dst: Optional[int] = None) -> None:
|
||||
"""Sends a tensor to the destination rank in a non-blocking way"""
|
||||
"""NOTE: `dst` is the local rank of the destination rank."""
|
||||
if dst is None:
|
||||
dst = self.next_rank
|
||||
|
||||
pynccl_comm = self.pynccl_comm
|
||||
if pynccl_comm is not None and not pynccl_comm.disabled:
|
||||
pynccl_comm.send(tensor, dst)
|
||||
else:
|
||||
torch.distributed.send(tensor, self.ranks[dst], self.device_group)
|
||||
|
||||
def recv(self,
|
||||
size: torch.Size,
|
||||
dtype: torch.dtype,
|
||||
src: Optional[int] = None) -> torch.Tensor:
|
||||
"""Receives a tensor from the src rank."""
|
||||
"""NOTE: `src` is the local rank of the destination rank."""
|
||||
if src is None:
|
||||
src = self.prev_rank
|
||||
|
||||
tensor = torch.empty(size, dtype=dtype, device=self.device)
|
||||
pynccl_comm = self.pynccl_comm
|
||||
if pynccl_comm is not None and not pynccl_comm.disabled:
|
||||
pynccl_comm.recv(tensor, src)
|
||||
else:
|
||||
torch.distributed.recv(tensor, self.ranks[src], self.device_group)
|
||||
return tensor
|
||||
|
||||
def destroy(self):
|
||||
if self.device_group is not None:
|
||||
torch.distributed.destroy_process_group(self.device_group)
|
||||
|
||||
Reference in New Issue
Block a user