[V1] Support MP Executor for multi node distributed inference (#23691)

Signed-off-by: Lu Fang <fanglu@fb.com>
Signed-off-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
Signed-off-by: Lucia Fang <fanglu@fb.com>
Signed-off-by: Lucia Fang <116399278+luccafong@users.noreply.github.com>
Signed-off-by: Nick Hill <nhill@redhat.com>
Co-authored-by: Nick Hill <nhill@redhat.com>
This commit is contained in:
Lucia Fang
2025-11-16 09:01:21 +00:00
committed by GitHub
co-authored by Nick Hill
parent a55b64635c
commit b316ac6589
10 changed files with 930 additions and 82 deletions
@@ -8,7 +8,7 @@ from dataclasses import dataclass, field
from multiprocessing import shared_memory
from pickle import PickleBuffer
from threading import Event
from typing import TYPE_CHECKING, Any
from typing import TYPE_CHECKING, Any, cast
from unittest.mock import patch
import torch
@@ -602,13 +602,87 @@ class MessageQueue:
return obj
return self.dequeue()
@staticmethod
def create_from_process_group_single_reader(
pg: ProcessGroup,
max_chunk_bytes,
max_chunks,
reader_rank: int = 0,
blocking: bool = False,
) -> tuple["MessageQueue", list[Handle]]:
"""
Creates a MessageQueue for a process group with a single reader.
This method is designed for scenarios where only one process (the reader)
will consume messages, and all other processes are writers. It sets up
the shared memory buffer and communication handles accordingly, and
gathers the handles from all processes to the reader.
Args:
pg (ProcessGroup): The torch distributed process group.
max_chunk_bytes (int): Maximum size in bytes for each chunk in the buffer.
max_chunks (int): Maximum number of chunks in the buffer.
reader_rank (int, optional): The global rank that will act as the reader.
Defaults to 0.
blocking (bool, optional): If True, blocks until all processes are ready.
Defaults to False.
Returns:
tuple[MessageQueue, list[Handle]]:
The MessageQueue instance for the calling process,
and a list of handles (only non-empty for the reader process).
"""
local_size = torch.cuda.device_count()
rank = dist.get_rank()
same_node = rank // local_size == reader_rank // local_size
buffer_io = MessageQueue(
n_reader=1,
n_local_reader=1 if same_node else 0,
max_chunk_bytes=max_chunk_bytes,
max_chunks=max_chunks,
)
handle = buffer_io.export_handle()
handles = [None] * dist.get_world_size(pg) if rank == reader_rank else None
dist.gather_object(handle, handles, dst=reader_rank, group=pg)
if blocking:
buffer_io.wait_until_ready()
return buffer_io, cast(list[Handle], handles or [])
@staticmethod
def create_from_process_group(
pg: ProcessGroup | StatelessProcessGroup,
max_chunk_bytes,
max_chunks,
writer_rank=0,
writer_rank: int = 0,
external_writer_handle=None,
blocking: bool = True,
) -> "MessageQueue":
"""
Creates a MessageQueue for a distributed process group with one writer and
multiple readers.
This method is designed for scenarios where one process (the writer) sends
messages, and all other processes (the readers) receive messages. It sets up
the shared memory buffer and socket communication handles accordingly, and
broadcasts the handle from the writer to all readers.
Args:
pg (ProcessGroup | StatelessProcessGroup): The torch distributed process
group.
max_chunk_bytes (int): Maximum size in bytes for each chunk in the buffer.
max_chunks (int): Maximum number of chunks in the buffer.
writer_rank (int, optional): The global rank that will act as the writer.
Defaults to 0.
external_writer_handle (Handle, optional): Used when there is a handle
from an external Message Queue. If provided, use this handle to init
PG writer message queue instead of creating a new one. Defaults to None.
blocking (bool, optional): If True, blocks until all processes are ready.
Defaults to True.
Returns:
MessageQueue: The MessageQueue instance for the calling process.
"""
if isinstance(pg, ProcessGroup):
group_rank = dist.get_rank(pg)
group_world_size = dist.get_world_size(pg)
@@ -617,23 +691,26 @@ class MessageQueue:
group_rank = pg.rank
group_world_size = pg.world_size
global_ranks = list(range(pg.world_size))
from vllm.distributed.parallel_state import in_the_same_node_as
status = in_the_same_node_as(pg, source_rank=writer_rank)
same_node_ranks = [i for i, s in enumerate(status) if s]
n_reader = group_world_size - 1
n_local_reader = len(same_node_ranks) - 1
local_reader_ranks = [i for i in same_node_ranks if i != writer_rank]
buffer_io: MessageQueue
if group_rank == writer_rank:
buffer_io = MessageQueue(
n_reader=n_reader,
n_local_reader=n_local_reader,
local_reader_ranks=local_reader_ranks,
max_chunk_bytes=max_chunk_bytes,
max_chunks=max_chunks,
)
if external_writer_handle is not None:
buffer_io = MessageQueue.create_from_handle(
external_writer_handle, group_rank
)
else:
same_node_ranks = [i for i, s in enumerate(status) if s]
n_reader = group_world_size - 1
n_local_reader = len(same_node_ranks) - 1
local_reader_ranks = [i for i in same_node_ranks if i != writer_rank]
buffer_io = MessageQueue(
n_reader=n_reader,
n_local_reader=n_local_reader,
local_reader_ranks=local_reader_ranks,
max_chunk_bytes=max_chunk_bytes,
max_chunks=max_chunks,
)
handle = buffer_io.export_handle()
if isinstance(pg, ProcessGroup):
dist.broadcast_object_list(
@@ -651,5 +728,6 @@ class MessageQueue:
else:
handle = pg.broadcast_obj(None, writer_rank)
buffer_io = MessageQueue.create_from_handle(handle, group_rank)
buffer_io.wait_until_ready()
if blocking:
buffer_io.wait_until_ready()
return buffer_io
+73 -4
View File
@@ -385,6 +385,33 @@ class GroupCoordinator:
torch.ops._C, "init_shm_manager"
)
def create_mq_broadcaster(
self, writer_rank=0, external_writer_handle=None, blocking=True
):
from vllm.distributed.device_communicators.shm_broadcast import MessageQueue
return MessageQueue.create_from_process_group(
self.cpu_group,
1 << 22,
6,
writer_rank=writer_rank,
external_writer_handle=external_writer_handle,
blocking=blocking,
)
def create_single_reader_mq_broadcasters(
self, reader_rank_in_group=0, blocking=False
):
from vllm.distributed.device_communicators.shm_broadcast import MessageQueue
return MessageQueue.create_from_process_group_single_reader(
self.cpu_group,
1 << 22,
6,
reader_rank=self.ranks[reader_rank_in_group],
blocking=blocking,
)
@property
def first_rank(self):
"""Return the global rank of the first process in the group"""
@@ -997,6 +1024,7 @@ class GroupCoordinator:
_WORLD: GroupCoordinator | None = None
_INNER_DP_WORLD: GroupCoordinator | None = None
_NODE_COUNT: int | None = None
@@ -1005,6 +1033,11 @@ def get_world_group() -> GroupCoordinator:
return _WORLD
def get_inner_dp_world_group() -> GroupCoordinator:
assert _INNER_DP_WORLD is not None, "inner dp world group is not initialized"
return _INNER_DP_WORLD
def init_world_group(
ranks: list[int], local_rank: int, backend: str
) -> GroupCoordinator:
@@ -1023,12 +1056,13 @@ def init_model_parallel_group(
backend: str,
use_message_queue_broadcaster: bool = False,
group_name: str | None = None,
use_device_communicator: bool = True,
) -> GroupCoordinator:
return GroupCoordinator(
group_ranks=group_ranks,
local_rank=local_rank,
torch_distributed_backend=backend,
use_device_communicator=True,
use_device_communicator=use_device_communicator,
use_message_queue_broadcaster=use_message_queue_broadcaster,
group_name=group_name,
)
@@ -1143,7 +1177,14 @@ def init_distributed_environment(
from vllm.config import get_current_vllm_config
config = get_current_vllm_config()
if (
if config is not None and config.parallel_config.nnodes > 1:
parallel_config = config.parallel_config
ip = parallel_config.master_addr
rank = parallel_config.data_parallel_rank * world_size + rank
world_size = parallel_config.world_size_across_dp
port = parallel_config.master_port
distributed_init_method = get_distributed_init_method(ip, port)
elif (
config is not None
and config.parallel_config.data_parallel_size > 1
and config.parallel_config.distributed_executor_backend != "external_launcher"
@@ -1164,6 +1205,14 @@ def init_distributed_environment(
distributed_init_method,
)
if not torch.distributed.is_initialized():
logger.info(
"world_size=%d rank=%d local_rank=%d distributed_init_method=%s backend=%s",
world_size,
rank,
local_rank,
distributed_init_method,
backend,
)
assert distributed_init_method is not None, (
"distributed_init_method must be provided when initializing "
"distributed environment"
@@ -1192,16 +1241,36 @@ def init_distributed_environment(
# local rank not set, this usually happens in single-node
# setting, where we can use rank as local rank
local_rank = envs.LOCAL_RANK if distributed_init_method == "env://" else rank
global _WORLD, _NODE_COUNT
global _WORLD, _NODE_COUNT, _INNER_DP_WORLD
if _WORLD is None:
ranks = list(range(torch.distributed.get_world_size()))
_WORLD = init_world_group(ranks, local_rank, backend)
_NODE_COUNT = _node_count(_WORLD.cpu_group)
if config.parallel_config.nnodes > 1:
_NODE_COUNT = config.parallel_config.nnodes
else:
_NODE_COUNT = _node_count(_WORLD.cpu_group)
logger.debug("Detected %d nodes in the distributed environment", _NODE_COUNT)
else:
assert _WORLD.world_size == torch.distributed.get_world_size(), (
"world group already initialized with a different world size"
)
if config.parallel_config.nnodes_within_dp > 1:
if parallel_config.data_parallel_size > 1:
world_size_inner_dp = parallel_config.world_size
group_ranks = [
[dp_rank * world_size_inner_dp + i for i in range(world_size_inner_dp)]
for dp_rank in range(parallel_config.data_parallel_size)
]
_INNER_DP_WORLD = init_model_parallel_group(
group_ranks,
get_world_group().local_rank,
backend,
use_message_queue_broadcaster=True,
group_name="inner_dp_world",
use_device_communicator=False,
)
else:
_INNER_DP_WORLD = _WORLD
def initialize_model_parallel(