[BugFix] Pick the DP world-group port at bind time via the coordination store (#51018)
Signed-off-by: aoshen02 <aoshen02@users.noreply.github.com> Signed-off-by: aoshen02 <aoshen@inferact.ai> Co-authored-by: aoshen02 <aoshen02@users.noreply.github.com> Co-authored-by: Claude <noreply@anthropic.com> Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
co-authored by
aoshen02
Claude
Codex
parent
a4b14e57b6
commit
caff2b44cb
@@ -13,8 +13,9 @@ from unittest.mock import Mock
|
||||
import pytest
|
||||
import ray
|
||||
import zmq
|
||||
from torch.distributed import TCPStore
|
||||
|
||||
from vllm.utils.network_utils import make_zmq_socket, split_zmq_path
|
||||
from vllm.utils.network_utils import get_open_port, make_zmq_socket, split_zmq_path
|
||||
from vllm.v1.engine.core import EngineCoreActorMixin
|
||||
from vllm.v1.engine.core_client import BackgroundResources
|
||||
from vllm.v1.engine.utils import (
|
||||
@@ -134,6 +135,72 @@ def _make_addresses() -> EngineZmqAddresses:
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("overrides", "expect_store"),
|
||||
[
|
||||
({"data_parallel_backend": "mp"}, True),
|
||||
({}, True),
|
||||
(
|
||||
{
|
||||
"data_parallel_backend": "mp",
|
||||
"data_parallel_rank": 1,
|
||||
"local_engines_only": True,
|
||||
},
|
||||
False,
|
||||
),
|
||||
({"data_parallel_backend": "mp", "data_parallel_rank_local": 0}, False),
|
||||
({"data_parallel_backend": "mp", "enable_elastic_ep": True}, False),
|
||||
({"data_parallel_backend": "mp", "data_parallel_size": 1}, False),
|
||||
],
|
||||
ids=["mp", "ray", "non-master-node", "offline", "elastic-ep", "single-engine"],
|
||||
)
|
||||
def test_coordination_store_held_only_by_the_online_dp_master(
|
||||
monkeypatch: pytest.MonkeyPatch, overrides: dict[str, Any], expect_store: bool
|
||||
) -> None:
|
||||
"""The DP master node holds a coordination store for both engine backends,
|
||||
so engines there pick world-group ports at bind time. Every other
|
||||
deployment keeps the pre-allocated ports and must not hold one."""
|
||||
|
||||
class FakeManager:
|
||||
def __init__(self, **kwargs) -> None:
|
||||
self.kwargs = kwargs
|
||||
|
||||
vllm_config = _make_vllm_config_ray_dp_multinode()
|
||||
parallel_config = vllm_config.parallel_config
|
||||
for name, value in overrides.items():
|
||||
setattr(parallel_config, name, value)
|
||||
# A fresh port per case so a lingering handshake socket cannot fail the next.
|
||||
parallel_config.data_parallel_rpc_port = get_open_port()
|
||||
parallel_config._coord_store_port = 0
|
||||
monkeypatch.setattr("vllm.v1.engine.utils.CoreEngineProcManager", FakeManager)
|
||||
monkeypatch.setattr("vllm.v1.engine.utils.CoreEngineActorManager", FakeManager)
|
||||
monkeypatch.setattr(
|
||||
"vllm.v1.engine.utils.wait_for_engine_startup", lambda *args, **kwargs: None
|
||||
)
|
||||
|
||||
with launch_core_engines(
|
||||
vllm_config,
|
||||
executor_class=_DummyExecutor,
|
||||
log_stats=False,
|
||||
addresses=_make_addresses(),
|
||||
) as engine_launch:
|
||||
assert engine_launch.engine_manager is not None
|
||||
if not expect_store:
|
||||
assert not parallel_config._coord_store_port
|
||||
return
|
||||
assert parallel_config._coord_store_port
|
||||
# Engines look the store up while they start, so it must be reachable
|
||||
# for as long as this frame is alive.
|
||||
client = TCPStore(
|
||||
parallel_config.data_parallel_master_ip,
|
||||
parallel_config._coord_store_port,
|
||||
is_master=False,
|
||||
wait_for_workers=False,
|
||||
)
|
||||
client.set("probe", b"1")
|
||||
assert client.get("probe") == b"1"
|
||||
|
||||
|
||||
def _make_cpu_placement_group():
|
||||
pg = ray.util.placement_group(
|
||||
[{"CPU": 0.001}, {"CPU": 1.0}],
|
||||
|
||||
@@ -0,0 +1,109 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""The DP world group picks its port when it binds, not ahead of time."""
|
||||
|
||||
import multiprocessing as mp
|
||||
import socket
|
||||
import traceback
|
||||
from multiprocessing.queues import Queue
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.distributed.utils import create_tcp_store
|
||||
from vllm.utils.network_utils import get_open_port
|
||||
|
||||
RESULT_TIMEOUT_S = 240
|
||||
|
||||
|
||||
def _join_dp_world_group(
|
||||
dp_rank: int, coord_store_port: int, dp_master_port: int, result: Queue
|
||||
) -> None:
|
||||
"""Bring up one DP rank of the world group and report what it saw."""
|
||||
try:
|
||||
import torch.distributed as dist
|
||||
|
||||
from vllm.config import VllmConfig, set_current_vllm_config
|
||||
from vllm.config.parallel import ParallelConfig
|
||||
from vllm.distributed.parallel_state import init_distributed_environment
|
||||
|
||||
parallel_config = ParallelConfig(
|
||||
data_parallel_size=2, data_parallel_rank=dp_rank
|
||||
)
|
||||
parallel_config.data_parallel_master_ip = "127.0.0.1"
|
||||
parallel_config._data_parallel_master_port_list = [dp_master_port]
|
||||
parallel_config._coord_store_port = coord_store_port
|
||||
|
||||
with set_current_vllm_config(VllmConfig(parallel_config=parallel_config)):
|
||||
init_distributed_environment(
|
||||
world_size=1, rank=0, local_rank=0, backend="gloo"
|
||||
)
|
||||
|
||||
reduced = torch.tensor([dp_rank + 1])
|
||||
dist.all_reduce(reduced)
|
||||
result.put((dp_rank, int(reduced.item()), None))
|
||||
except Exception:
|
||||
result.put((dp_rank, None, traceback.format_exc()))
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not torch.distributed.is_gloo_available(), reason="gloo backend required"
|
||||
)
|
||||
@pytest.mark.parametrize("with_coord_store", [True, False])
|
||||
def test_dp_world_group_survives_a_taken_preallocated_port(with_coord_store: bool):
|
||||
"""With a coordination store, the two ranks agree on a port the group binds
|
||||
itself, so a pre-allocated port taken in the meantime cannot stop startup.
|
||||
Without one, the pre-allocated port is still used as before.
|
||||
"""
|
||||
squatter = None
|
||||
coord_store = None
|
||||
try:
|
||||
if with_coord_store:
|
||||
# Something else grabbed the pre-allocated port after it was picked.
|
||||
squatter = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
||||
squatter.bind(("127.0.0.1", 0))
|
||||
squatter.listen(1)
|
||||
dp_master_port = squatter.getsockname()[1]
|
||||
coord_store = create_tcp_store(
|
||||
"127.0.0.1", 0, is_master=True, world_size=-1, wait_for_workers=False
|
||||
)
|
||||
coord_store_port = coord_store.port
|
||||
else:
|
||||
dp_master_port = get_open_port()
|
||||
coord_store_port = 0
|
||||
|
||||
ctx = mp.get_context("spawn")
|
||||
result: Queue = ctx.Queue()
|
||||
procs = [
|
||||
ctx.Process(
|
||||
target=_join_dp_world_group,
|
||||
args=(dp_rank, coord_store_port, dp_master_port, result),
|
||||
)
|
||||
for dp_rank in range(2)
|
||||
]
|
||||
for proc in procs:
|
||||
proc.start()
|
||||
reports: list[tuple[int, int | None, str | None]] = []
|
||||
try:
|
||||
while len(reports) < len(procs):
|
||||
reports.append(result.get(timeout=RESULT_TIMEOUT_S))
|
||||
if reports[-1][2] is not None:
|
||||
break # the peer would wait on this rank; do not wait for it
|
||||
finally:
|
||||
for proc in procs:
|
||||
proc.join(timeout=30)
|
||||
if proc.is_alive():
|
||||
proc.kill()
|
||||
|
||||
for dp_rank, reduced, error in sorted(reports):
|
||||
assert error is None, f"DP rank {dp_rank} failed:\n{error}"
|
||||
# Both ranks joined the same group: 1 + 2 over the two ranks.
|
||||
assert reduced == 3, f"DP rank {dp_rank} all-reduced to {reduced}"
|
||||
|
||||
if with_coord_store:
|
||||
assert coord_store is not None
|
||||
bound_port = int(coord_store.get("world_pg_port").decode())
|
||||
assert bound_port != dp_master_port
|
||||
finally:
|
||||
if squatter is not None:
|
||||
squatter.close()
|
||||
Reference in New Issue
Block a user