[core][distributed] zmq fallback for broadcasting large objects (#6183)
[core][distributed] add zmq fallback for broadcasting large objects (#6183)
This commit is contained in:
@@ -124,7 +124,7 @@ class GroupCoordinator:
|
||||
# communicators are only created for world size > 1
|
||||
pynccl_comm: Optional[Any] # PyNccl communicator
|
||||
ca_comm: Optional[Any] # Custom allreduce communicator
|
||||
shm_broadcaster: Optional[Any] # shared memory broadcaster
|
||||
mq_broadcaster: Optional[Any] # shared memory broadcaster
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
@@ -133,6 +133,7 @@ class GroupCoordinator:
|
||||
torch_distributed_backend: Union[str, Backend],
|
||||
use_pynccl: bool,
|
||||
use_custom_allreduce: bool,
|
||||
use_message_queue_broadcaster: bool = False,
|
||||
):
|
||||
|
||||
self.rank = torch.distributed.get_rank()
|
||||
@@ -190,10 +191,10 @@ class GroupCoordinator:
|
||||
self.ca_comm = None
|
||||
|
||||
from vllm.distributed.device_communicators.shm_broadcast import (
|
||||
ShmRingBufferIO)
|
||||
self.shm_broadcaster: Optional[ShmRingBufferIO] = None
|
||||
if self.world_size > 1 and is_in_the_same_node(self.cpu_group):
|
||||
self.shm_broadcaster = ShmRingBufferIO.create_from_process_group(
|
||||
MessageQueue)
|
||||
self.mq_broadcaster: Optional[MessageQueue] = None
|
||||
if use_message_queue_broadcaster and self.world_size > 1:
|
||||
self.mq_broadcaster = MessageQueue.create_from_process_group(
|
||||
self.cpu_group, 1 << 22, 6)
|
||||
|
||||
@property
|
||||
@@ -377,9 +378,9 @@ class GroupCoordinator:
|
||||
# Bypass the function if we are using only 1 GPU.
|
||||
if self.world_size == 1:
|
||||
return obj
|
||||
if self.shm_broadcaster is not None:
|
||||
assert src == 0, "Shared memory broadcaster only supports src=0"
|
||||
return self.shm_broadcaster.broadcast_object(obj)
|
||||
if self.mq_broadcaster is not None:
|
||||
assert src == 0, "Message queue broadcaster only supports src=0"
|
||||
return self.mq_broadcaster.broadcast_object(obj)
|
||||
if self.rank_in_group == src:
|
||||
torch.distributed.broadcast_object_list([obj],
|
||||
src=self.ranks[src],
|
||||
@@ -696,8 +697,8 @@ class GroupCoordinator:
|
||||
self.pynccl_comm = None
|
||||
if self.ca_comm is not None:
|
||||
self.ca_comm = None
|
||||
if self.shm_broadcaster is not None:
|
||||
self.shm_broadcaster = None
|
||||
if self.mq_broadcaster is not None:
|
||||
self.mq_broadcaster = None
|
||||
|
||||
|
||||
_WORLD: Optional[GroupCoordinator] = None
|
||||
@@ -720,10 +721,12 @@ def init_world_group(ranks: List[int], local_rank: int,
|
||||
|
||||
|
||||
def init_model_parallel_group(
|
||||
group_ranks: List[List[int]],
|
||||
local_rank: int,
|
||||
backend: str,
|
||||
use_custom_allreduce: Optional[bool] = None) -> GroupCoordinator:
|
||||
group_ranks: List[List[int]],
|
||||
local_rank: int,
|
||||
backend: str,
|
||||
use_custom_allreduce: Optional[bool] = None,
|
||||
use_message_queue_broadcaster: bool = False,
|
||||
) -> GroupCoordinator:
|
||||
if use_custom_allreduce is None:
|
||||
use_custom_allreduce = _ENABLE_CUSTOM_ALL_REDUCE
|
||||
return GroupCoordinator(
|
||||
@@ -732,6 +735,7 @@ def init_model_parallel_group(
|
||||
torch_distributed_backend=backend,
|
||||
use_pynccl=True,
|
||||
use_custom_allreduce=use_custom_allreduce,
|
||||
use_message_queue_broadcaster=use_message_queue_broadcaster,
|
||||
)
|
||||
|
||||
|
||||
@@ -880,8 +884,12 @@ def initialize_model_parallel(
|
||||
range(i * tensor_model_parallel_size,
|
||||
(i + 1) * tensor_model_parallel_size))
|
||||
group_ranks.append(ranks)
|
||||
|
||||
# message queue broadcaster is only used in tensor model parallel group
|
||||
_TP = init_model_parallel_group(group_ranks,
|
||||
get_world_group().local_rank, backend)
|
||||
get_world_group().local_rank,
|
||||
backend,
|
||||
use_message_queue_broadcaster=True)
|
||||
|
||||
# Build the pipeline model-parallel groups.
|
||||
num_pipeline_model_parallel_groups: int = (world_size //
|
||||
@@ -993,15 +1001,15 @@ def destroy_distributed_environment():
|
||||
torch.distributed.destroy_process_group()
|
||||
|
||||
|
||||
def is_in_the_same_node(pg: ProcessGroup):
|
||||
def in_the_same_node_as(pg: ProcessGroup, source_rank: int = 0) -> List[bool]:
|
||||
"""
|
||||
This is a collective operation that checks if all processes in the group
|
||||
are in the same node. It tests if all processes are attached to the same
|
||||
This is a collective operation that returns if each rank is in the same node
|
||||
as the source rank. It tests if processes are attached to the same
|
||||
memory system (shared access to shared memory).
|
||||
"""
|
||||
assert torch.distributed.get_backend(
|
||||
pg) != torch.distributed.Backend.NCCL, (
|
||||
"is_in_the_same_node should be tested with a non-NCCL group.")
|
||||
"in_the_same_node_as should be tested with a non-NCCL group.")
|
||||
# local rank inside the group
|
||||
rank = torch.distributed.get_rank(group=pg)
|
||||
world_size = torch.distributed.get_world_size(group=pg)
|
||||
@@ -1017,19 +1025,19 @@ def is_in_the_same_node(pg: ProcessGroup):
|
||||
|
||||
try:
|
||||
with contextlib.suppress(OSError):
|
||||
if rank == 0:
|
||||
if rank == source_rank:
|
||||
# create a shared memory segment
|
||||
shm = shared_memory.SharedMemory(create=True, size=128)
|
||||
shm.buf[:len(magic_message)] = magic_message
|
||||
torch.distributed.broadcast_object_list([shm.name],
|
||||
src=ranks[0],
|
||||
src=ranks[source_rank],
|
||||
group=pg)
|
||||
is_in_the_same_node[0] = 1
|
||||
is_in_the_same_node[rank] = 1
|
||||
else:
|
||||
# try to open the shared memory segment
|
||||
recv = [None]
|
||||
torch.distributed.broadcast_object_list(recv,
|
||||
src=ranks[0],
|
||||
src=ranks[source_rank],
|
||||
group=pg)
|
||||
name = recv[0]
|
||||
# fix to https://stackoverflow.com/q/62748654/9191338
|
||||
@@ -1050,8 +1058,8 @@ def is_in_the_same_node(pg: ProcessGroup):
|
||||
|
||||
# clean up the shared memory segment
|
||||
with contextlib.suppress(OSError):
|
||||
if rank == 0 and shm:
|
||||
if rank == source_rank and shm:
|
||||
shm.unlink()
|
||||
torch.distributed.all_reduce(is_in_the_same_node, group=pg)
|
||||
|
||||
return is_in_the_same_node.sum().item() == world_size
|
||||
return [x == 1 for x in is_in_the_same_node.tolist()]
|
||||
|
||||
Reference in New Issue
Block a user