[Bugfix] Seed RayExecutorV2 TCPStore port by DP rank to avoid collisions (#45960)
Signed-off-by: Seiji Eicher <seiji@anyscale.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
7a327f0b4f
commit
ded6676458
@@ -17,6 +17,7 @@ import ray
|
||||
from vllm import LLM
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.engine.arg_utils import EngineArgs
|
||||
from vllm.v1.executor import ray_executor_v2
|
||||
from vllm.v1.executor.ray_executor_v2 import RayExecutorV2
|
||||
|
||||
pytestmark = pytest.mark.usefixtures("enable_ray_v2_backend")
|
||||
@@ -95,6 +96,43 @@ def assert_executor(executor, tp_size, pp_size):
|
||||
assert handle.node_id is not None
|
||||
|
||||
|
||||
def test_select_tcpstore_port_seeds_disjoint_windows(monkeypatch):
|
||||
"""Co-located DP engines scan distinct, adjacent port windows, so two
|
||||
engines on a node cannot pick the same TCPStore port."""
|
||||
requested = []
|
||||
|
||||
def fake_get_open_port(start_port, max_attempts):
|
||||
requested.append((start_port, max_attempts))
|
||||
return start_port
|
||||
|
||||
monkeypatch.setattr(ray_executor_v2, "_get_open_port", fake_get_open_port)
|
||||
|
||||
ports = [
|
||||
RayExecutorV2._select_tcpstore_port(rank, master_port=29500)
|
||||
for rank in range(4)
|
||||
]
|
||||
|
||||
assert requested == [(29600, 32), (29632, 32), (29664, 32), (29696, 32)]
|
||||
assert len(set(ports)) == 4
|
||||
|
||||
|
||||
def test_select_tcpstore_port_non_dp_uses_random(monkeypatch):
|
||||
"""A non-DP engine has no local rank and uses a random port."""
|
||||
monkeypatch.setattr(ray_executor_v2, "get_open_port", lambda: 54321)
|
||||
assert RayExecutorV2._select_tcpstore_port(None, master_port=29500) == 54321
|
||||
|
||||
|
||||
def test_select_tcpstore_port_full_window_uses_random(monkeypatch):
|
||||
"""A fully occupied window falls back to a random port."""
|
||||
|
||||
def raise_full(start_port, max_attempts):
|
||||
raise RuntimeError("no open port")
|
||||
|
||||
monkeypatch.setattr(ray_executor_v2, "_get_open_port", raise_full)
|
||||
monkeypatch.setattr(ray_executor_v2, "get_open_port", lambda: 54321)
|
||||
assert RayExecutorV2._select_tcpstore_port(0, master_port=29500) == 54321
|
||||
|
||||
|
||||
@pytest.mark.parametrize("tp_size, pp_size", [(1, 1), (2, 1), (4, 1), (2, 2)])
|
||||
def test_ray_v2_executor(tp_size, pp_size):
|
||||
"""Validate RayExecutorV2 with various TP/PP configs."""
|
||||
|
||||
@@ -17,6 +17,7 @@ from vllm.distributed.device_communicators.shm_broadcast import (
|
||||
from vllm.logger import init_logger
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.network_utils import (
|
||||
_get_open_port,
|
||||
get_distributed_init_method,
|
||||
get_open_port,
|
||||
)
|
||||
@@ -259,6 +260,25 @@ class RayExecutorV2(MultiprocExecutor):
|
||||
return {"num_gpus": num_devices}
|
||||
return {"num_gpus": 0, "resources": {device_key: num_devices}}
|
||||
|
||||
@staticmethod
|
||||
def _select_tcpstore_port(local_dp_rank: int | None, master_port: int) -> int:
|
||||
"""Pick the torch.distributed TCPStore port for this engine.
|
||||
|
||||
Co-located DP engines choosing this port with a shared random search
|
||||
collide intermittently. Seeding by node-local DP rank gives each a
|
||||
disjoint window. Non-DP engines and full windows fall back to a
|
||||
random port.
|
||||
"""
|
||||
if local_dp_rank is None:
|
||||
return get_open_port()
|
||||
# Offset past the DP master port reserved range, one window per rank.
|
||||
window = 32
|
||||
start_port = master_port + 100 + local_dp_rank * window
|
||||
try:
|
||||
return _get_open_port(start_port=start_port, max_attempts=window)
|
||||
except RuntimeError:
|
||||
return get_open_port()
|
||||
|
||||
def _init_executor(self) -> None:
|
||||
"""Initialize the RayExecutorV2 executor."""
|
||||
self._finalizer = weakref.finalize(self, self.shutdown)
|
||||
@@ -308,7 +328,12 @@ class RayExecutorV2(MultiprocExecutor):
|
||||
# The TCPStore server runs on rank 0's node, so all workers
|
||||
# must be able to reach this address.
|
||||
dist_ip = bundle_assignments[0]["node_ip"]
|
||||
distributed_init_method = get_distributed_init_method(dist_ip, get_open_port())
|
||||
parallel_config = self.vllm_config.parallel_config
|
||||
port = self._select_tcpstore_port(
|
||||
parallel_config.data_parallel_rank_local,
|
||||
parallel_config.data_parallel_master_port,
|
||||
)
|
||||
distributed_init_method = get_distributed_init_method(dist_ip, port)
|
||||
|
||||
# Step 4: Create broadcast MessageQueue.
|
||||
# Workers on the driver node use shared memory; the rest use TCP.
|
||||
|
||||
Reference in New Issue
Block a user