[V1] Support MP Executor for multi node distributed inference (#23691)

Signed-off-by: Lu Fang <fanglu@fb.com>
Signed-off-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
Signed-off-by: Lucia Fang <fanglu@fb.com>
Signed-off-by: Lucia Fang <116399278+luccafong@users.noreply.github.com>
Signed-off-by: Nick Hill <nhill@redhat.com>
Co-authored-by: Nick Hill <nhill@redhat.com>
This commit is contained in:
Lucia Fang
2025-11-16 09:01:21 +00:00
committed by GitHub
co-authored by Nick Hill
parent a55b64635c
commit b316ac6589
10 changed files with 930 additions and 82 deletions
+73 -4
View File
@@ -385,6 +385,33 @@ class GroupCoordinator:
torch.ops._C, "init_shm_manager"
)
def create_mq_broadcaster(
self, writer_rank=0, external_writer_handle=None, blocking=True
):
from vllm.distributed.device_communicators.shm_broadcast import MessageQueue
return MessageQueue.create_from_process_group(
self.cpu_group,
1 << 22,
6,
writer_rank=writer_rank,
external_writer_handle=external_writer_handle,
blocking=blocking,
)
def create_single_reader_mq_broadcasters(
self, reader_rank_in_group=0, blocking=False
):
from vllm.distributed.device_communicators.shm_broadcast import MessageQueue
return MessageQueue.create_from_process_group_single_reader(
self.cpu_group,
1 << 22,
6,
reader_rank=self.ranks[reader_rank_in_group],
blocking=blocking,
)
@property
def first_rank(self):
"""Return the global rank of the first process in the group"""
@@ -997,6 +1024,7 @@ class GroupCoordinator:
_WORLD: GroupCoordinator | None = None
_INNER_DP_WORLD: GroupCoordinator | None = None
_NODE_COUNT: int | None = None
@@ -1005,6 +1033,11 @@ def get_world_group() -> GroupCoordinator:
return _WORLD
def get_inner_dp_world_group() -> GroupCoordinator:
assert _INNER_DP_WORLD is not None, "inner dp world group is not initialized"
return _INNER_DP_WORLD
def init_world_group(
ranks: list[int], local_rank: int, backend: str
) -> GroupCoordinator:
@@ -1023,12 +1056,13 @@ def init_model_parallel_group(
backend: str,
use_message_queue_broadcaster: bool = False,
group_name: str | None = None,
use_device_communicator: bool = True,
) -> GroupCoordinator:
return GroupCoordinator(
group_ranks=group_ranks,
local_rank=local_rank,
torch_distributed_backend=backend,
use_device_communicator=True,
use_device_communicator=use_device_communicator,
use_message_queue_broadcaster=use_message_queue_broadcaster,
group_name=group_name,
)
@@ -1143,7 +1177,14 @@ def init_distributed_environment(
from vllm.config import get_current_vllm_config
config = get_current_vllm_config()
if (
if config is not None and config.parallel_config.nnodes > 1:
parallel_config = config.parallel_config
ip = parallel_config.master_addr
rank = parallel_config.data_parallel_rank * world_size + rank
world_size = parallel_config.world_size_across_dp
port = parallel_config.master_port
distributed_init_method = get_distributed_init_method(ip, port)
elif (
config is not None
and config.parallel_config.data_parallel_size > 1
and config.parallel_config.distributed_executor_backend != "external_launcher"
@@ -1164,6 +1205,14 @@ def init_distributed_environment(
distributed_init_method,
)
if not torch.distributed.is_initialized():
logger.info(
"world_size=%d rank=%d local_rank=%d distributed_init_method=%s backend=%s",
world_size,
rank,
local_rank,
distributed_init_method,
backend,
)
assert distributed_init_method is not None, (
"distributed_init_method must be provided when initializing "
"distributed environment"
@@ -1192,16 +1241,36 @@ def init_distributed_environment(
# local rank not set, this usually happens in single-node
# setting, where we can use rank as local rank
local_rank = envs.LOCAL_RANK if distributed_init_method == "env://" else rank
global _WORLD, _NODE_COUNT
global _WORLD, _NODE_COUNT, _INNER_DP_WORLD
if _WORLD is None:
ranks = list(range(torch.distributed.get_world_size()))
_WORLD = init_world_group(ranks, local_rank, backend)
_NODE_COUNT = _node_count(_WORLD.cpu_group)
if config.parallel_config.nnodes > 1:
_NODE_COUNT = config.parallel_config.nnodes
else:
_NODE_COUNT = _node_count(_WORLD.cpu_group)
logger.debug("Detected %d nodes in the distributed environment", _NODE_COUNT)
else:
assert _WORLD.world_size == torch.distributed.get_world_size(), (
"world group already initialized with a different world size"
)
if config.parallel_config.nnodes_within_dp > 1:
if parallel_config.data_parallel_size > 1:
world_size_inner_dp = parallel_config.world_size
group_ranks = [
[dp_rank * world_size_inner_dp + i for i in range(world_size_inner_dp)]
for dp_rank in range(parallel_config.data_parallel_size)
]
_INNER_DP_WORLD = init_model_parallel_group(
group_ranks,
get_world_group().local_rank,
backend,
use_message_queue_broadcaster=True,
group_name="inner_dp_world",
use_device_communicator=False,
)
else:
_INNER_DP_WORLD = _WORLD
def initialize_model_parallel(