[V1] Support MP Executor for multi node distributed inference (#23691)
Signed-off-by: Lu Fang <fanglu@fb.com> Signed-off-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com> Signed-off-by: Lucia Fang <fanglu@fb.com> Signed-off-by: Lucia Fang <116399278+luccafong@users.noreply.github.com> Signed-off-by: Nick Hill <nhill@redhat.com> Co-authored-by: Nick Hill <nhill@redhat.com>
This commit is contained in:
@@ -385,6 +385,33 @@ class GroupCoordinator:
|
||||
torch.ops._C, "init_shm_manager"
|
||||
)
|
||||
|
||||
def create_mq_broadcaster(
|
||||
self, writer_rank=0, external_writer_handle=None, blocking=True
|
||||
):
|
||||
from vllm.distributed.device_communicators.shm_broadcast import MessageQueue
|
||||
|
||||
return MessageQueue.create_from_process_group(
|
||||
self.cpu_group,
|
||||
1 << 22,
|
||||
6,
|
||||
writer_rank=writer_rank,
|
||||
external_writer_handle=external_writer_handle,
|
||||
blocking=blocking,
|
||||
)
|
||||
|
||||
def create_single_reader_mq_broadcasters(
|
||||
self, reader_rank_in_group=0, blocking=False
|
||||
):
|
||||
from vllm.distributed.device_communicators.shm_broadcast import MessageQueue
|
||||
|
||||
return MessageQueue.create_from_process_group_single_reader(
|
||||
self.cpu_group,
|
||||
1 << 22,
|
||||
6,
|
||||
reader_rank=self.ranks[reader_rank_in_group],
|
||||
blocking=blocking,
|
||||
)
|
||||
|
||||
@property
|
||||
def first_rank(self):
|
||||
"""Return the global rank of the first process in the group"""
|
||||
@@ -997,6 +1024,7 @@ class GroupCoordinator:
|
||||
|
||||
|
||||
_WORLD: GroupCoordinator | None = None
|
||||
_INNER_DP_WORLD: GroupCoordinator | None = None
|
||||
_NODE_COUNT: int | None = None
|
||||
|
||||
|
||||
@@ -1005,6 +1033,11 @@ def get_world_group() -> GroupCoordinator:
|
||||
return _WORLD
|
||||
|
||||
|
||||
def get_inner_dp_world_group() -> GroupCoordinator:
|
||||
assert _INNER_DP_WORLD is not None, "inner dp world group is not initialized"
|
||||
return _INNER_DP_WORLD
|
||||
|
||||
|
||||
def init_world_group(
|
||||
ranks: list[int], local_rank: int, backend: str
|
||||
) -> GroupCoordinator:
|
||||
@@ -1023,12 +1056,13 @@ def init_model_parallel_group(
|
||||
backend: str,
|
||||
use_message_queue_broadcaster: bool = False,
|
||||
group_name: str | None = None,
|
||||
use_device_communicator: bool = True,
|
||||
) -> GroupCoordinator:
|
||||
return GroupCoordinator(
|
||||
group_ranks=group_ranks,
|
||||
local_rank=local_rank,
|
||||
torch_distributed_backend=backend,
|
||||
use_device_communicator=True,
|
||||
use_device_communicator=use_device_communicator,
|
||||
use_message_queue_broadcaster=use_message_queue_broadcaster,
|
||||
group_name=group_name,
|
||||
)
|
||||
@@ -1143,7 +1177,14 @@ def init_distributed_environment(
|
||||
from vllm.config import get_current_vllm_config
|
||||
|
||||
config = get_current_vllm_config()
|
||||
if (
|
||||
if config is not None and config.parallel_config.nnodes > 1:
|
||||
parallel_config = config.parallel_config
|
||||
ip = parallel_config.master_addr
|
||||
rank = parallel_config.data_parallel_rank * world_size + rank
|
||||
world_size = parallel_config.world_size_across_dp
|
||||
port = parallel_config.master_port
|
||||
distributed_init_method = get_distributed_init_method(ip, port)
|
||||
elif (
|
||||
config is not None
|
||||
and config.parallel_config.data_parallel_size > 1
|
||||
and config.parallel_config.distributed_executor_backend != "external_launcher"
|
||||
@@ -1164,6 +1205,14 @@ def init_distributed_environment(
|
||||
distributed_init_method,
|
||||
)
|
||||
if not torch.distributed.is_initialized():
|
||||
logger.info(
|
||||
"world_size=%d rank=%d local_rank=%d distributed_init_method=%s backend=%s",
|
||||
world_size,
|
||||
rank,
|
||||
local_rank,
|
||||
distributed_init_method,
|
||||
backend,
|
||||
)
|
||||
assert distributed_init_method is not None, (
|
||||
"distributed_init_method must be provided when initializing "
|
||||
"distributed environment"
|
||||
@@ -1192,16 +1241,36 @@ def init_distributed_environment(
|
||||
# local rank not set, this usually happens in single-node
|
||||
# setting, where we can use rank as local rank
|
||||
local_rank = envs.LOCAL_RANK if distributed_init_method == "env://" else rank
|
||||
global _WORLD, _NODE_COUNT
|
||||
global _WORLD, _NODE_COUNT, _INNER_DP_WORLD
|
||||
if _WORLD is None:
|
||||
ranks = list(range(torch.distributed.get_world_size()))
|
||||
_WORLD = init_world_group(ranks, local_rank, backend)
|
||||
_NODE_COUNT = _node_count(_WORLD.cpu_group)
|
||||
if config.parallel_config.nnodes > 1:
|
||||
_NODE_COUNT = config.parallel_config.nnodes
|
||||
else:
|
||||
_NODE_COUNT = _node_count(_WORLD.cpu_group)
|
||||
logger.debug("Detected %d nodes in the distributed environment", _NODE_COUNT)
|
||||
else:
|
||||
assert _WORLD.world_size == torch.distributed.get_world_size(), (
|
||||
"world group already initialized with a different world size"
|
||||
)
|
||||
if config.parallel_config.nnodes_within_dp > 1:
|
||||
if parallel_config.data_parallel_size > 1:
|
||||
world_size_inner_dp = parallel_config.world_size
|
||||
group_ranks = [
|
||||
[dp_rank * world_size_inner_dp + i for i in range(world_size_inner_dp)]
|
||||
for dp_rank in range(parallel_config.data_parallel_size)
|
||||
]
|
||||
_INNER_DP_WORLD = init_model_parallel_group(
|
||||
group_ranks,
|
||||
get_world_group().local_rank,
|
||||
backend,
|
||||
use_message_queue_broadcaster=True,
|
||||
group_name="inner_dp_world",
|
||||
use_device_communicator=False,
|
||||
)
|
||||
else:
|
||||
_INNER_DP_WORLD = _WORLD
|
||||
|
||||
|
||||
def initialize_model_parallel(
|
||||
|
||||
Reference in New Issue
Block a user