[BugFix] Bind RayExecutorV2 TCPStore before publishing its port (#50969)
Signed-off-by: aoshen02 <aoshen02@users.noreply.github.com> Signed-off-by: aoshen02 <aoshen@inferact.ai> Co-authored-by: aoshen02 <aoshen02@users.noreply.github.com> Co-authored-by: Claude <noreply@anthropic.com> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
co-authored by
aoshen02
Claude
OpenAI Codex
Andreas Karatzas
parent
2f1cba799e
commit
67e86d1e6f
@@ -7,12 +7,17 @@ Validates executor initialization, placement group support, RPC calls,
|
||||
and distributed execution with various TP/PP configurations.
|
||||
"""
|
||||
|
||||
import errno
|
||||
import gc
|
||||
import socket
|
||||
import threading
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import patch
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
import pytest
|
||||
import ray
|
||||
from torch.distributed import TCPStore
|
||||
|
||||
from vllm import LLM
|
||||
from vllm.config import VllmConfig
|
||||
@@ -102,67 +107,30 @@ def assert_executor(executor, tp_size, pp_size):
|
||||
assert handle.node_id is not None
|
||||
|
||||
|
||||
def test_select_tcpstore_port_seeds_disjoint_windows(monkeypatch):
|
||||
"""Co-located DP engines scan distinct, adjacent port windows, so two
|
||||
engines on a node cannot pick the same TCPStore port."""
|
||||
requested = []
|
||||
def test_dist_init_store_owns_port(monkeypatch):
|
||||
"""The rank-0 actor keeps the selected TCPStore port bound."""
|
||||
worker = ray_executor_v2.RayWorkerProc.__new__(ray_executor_v2.RayWorkerProc)
|
||||
worker._parallel_config = SimpleNamespace(world_size=1)
|
||||
monkeypatch.setattr(ray.util, "get_node_ip_address", lambda: "127.0.0.1")
|
||||
|
||||
def fake_get_open_port(start_port, max_attempts):
|
||||
requested.append((start_port, max_attempts))
|
||||
return start_port
|
||||
init_method = worker.create_dist_init_method()
|
||||
port = urlsplit(init_method).port
|
||||
assert port is not None and port != 0
|
||||
|
||||
monkeypatch.setattr(ray_executor_v2, "_get_open_port", fake_get_open_port)
|
||||
with socket.socket() as contender, pytest.raises(OSError) as exc_info:
|
||||
contender.bind(("127.0.0.1", port))
|
||||
assert exc_info.value.errno == errno.EADDRINUSE
|
||||
|
||||
ports = [
|
||||
RayExecutorV2._select_tcpstore_port(rank, master_port=29500)
|
||||
for rank in range(4)
|
||||
]
|
||||
|
||||
assert requested == [(29600, 32), (29632, 32), (29664, 32), (29696, 32)]
|
||||
assert len(set(ports)) == 4
|
||||
|
||||
|
||||
def test_select_tcpstore_port_non_dp_uses_random(monkeypatch):
|
||||
"""A non-DP engine has no local rank and uses a random port."""
|
||||
monkeypatch.setattr(ray_executor_v2, "get_open_port", lambda: 54321)
|
||||
assert RayExecutorV2._select_tcpstore_port(None, master_port=29500) == 54321
|
||||
|
||||
|
||||
def test_colocated_tcpstore_ports_differ():
|
||||
"""Two or more replicas on one host can collide on the same TCPStore port.
|
||||
Their ports must differ and never be a low system port.
|
||||
"""
|
||||
ports = []
|
||||
real_select = RayExecutorV2._select_tcpstore_port
|
||||
|
||||
def record(local_dp_rank, master_port):
|
||||
port = real_select(local_dp_rank, master_port)
|
||||
ports.append(port)
|
||||
return port
|
||||
|
||||
with patch.object(RayExecutorV2, "_select_tcpstore_port", staticmethod(record)):
|
||||
first = RayExecutorV2(vllm_config=create_vllm_config())
|
||||
try:
|
||||
second = RayExecutorV2(vllm_config=create_vllm_config())
|
||||
try:
|
||||
assert len(ports) == 2
|
||||
assert min(ports) >= 1024
|
||||
assert ports[0] != ports[1]
|
||||
finally:
|
||||
second.shutdown()
|
||||
finally:
|
||||
first.shutdown()
|
||||
|
||||
|
||||
def test_select_tcpstore_port_full_window_uses_random(monkeypatch):
|
||||
"""A fully occupied window falls back to a random port."""
|
||||
|
||||
def raise_full(start_port, max_attempts):
|
||||
raise RuntimeError("no open port")
|
||||
|
||||
monkeypatch.setattr(ray_executor_v2, "_get_open_port", raise_full)
|
||||
monkeypatch.setattr(ray_executor_v2, "get_open_port", lambda: 54321)
|
||||
assert RayExecutorV2._select_tcpstore_port(0, master_port=29500) == 54321
|
||||
reused_store = TCPStore(
|
||||
"127.0.0.1",
|
||||
port,
|
||||
world_size=1,
|
||||
is_master=True,
|
||||
wait_for_workers=False,
|
||||
multi_tenant=True,
|
||||
)
|
||||
worker._dist_init_store.set("key", "value")
|
||||
assert reused_store.get("key") == b"value"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("tp_size, pp_size", [(1, 1), (2, 1), (4, 1), (2, 2)])
|
||||
|
||||
@@ -8,6 +8,8 @@ from collections import defaultdict, deque
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
from torch.distributed import TCPStore
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed.device_communicators.shm_broadcast import (
|
||||
@@ -17,9 +19,7 @@ from vllm.distributed.device_communicators.shm_broadcast import (
|
||||
from vllm.logger import init_logger
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.network_utils import (
|
||||
_get_open_port,
|
||||
get_distributed_init_method,
|
||||
get_open_port,
|
||||
)
|
||||
from vllm.v1.executor.multiproc_executor import (
|
||||
FutureWrapper,
|
||||
@@ -106,22 +106,35 @@ class RayWorkerProc(WorkerProc):
|
||||
self,
|
||||
vllm_config: VllmConfig,
|
||||
rank: int,
|
||||
distributed_init_method: str,
|
||||
input_shm_handle: Handle,
|
||||
is_driver_worker: bool,
|
||||
is_driver_node: bool = False,
|
||||
):
|
||||
# Defer WorkerProc.__init__ until GPU IDs are known.
|
||||
self._is_driver_node = is_driver_node
|
||||
self._parallel_config = vllm_config.parallel_config
|
||||
self._init_kwargs = dict(
|
||||
vllm_config=vllm_config,
|
||||
rank=rank,
|
||||
distributed_init_method=distributed_init_method,
|
||||
input_shm_handle=input_shm_handle,
|
||||
shared_worker_lock=None,
|
||||
is_driver_worker=is_driver_worker,
|
||||
)
|
||||
|
||||
def create_dist_init_method(self) -> str:
|
||||
host = ray.util.get_node_ip_address()
|
||||
store = TCPStore(
|
||||
host_name=host,
|
||||
port=0,
|
||||
world_size=self._parallel_config.world_size,
|
||||
is_master=True,
|
||||
wait_for_workers=False,
|
||||
multi_tenant=True,
|
||||
)
|
||||
# Keep the bound server alive until init_process_group reuses it.
|
||||
self._dist_init_store = store
|
||||
return get_distributed_init_method(host, store.port)
|
||||
|
||||
def get_node_and_physical_gpu_ids(self) -> tuple[str, list[int]]:
|
||||
"""Return (node_id, physical_gpu_ids) assigned to this actor by Ray."""
|
||||
node_id = ray.get_runtime_context().get_node_id()
|
||||
@@ -140,6 +153,7 @@ class RayWorkerProc(WorkerProc):
|
||||
self,
|
||||
local_rank: int,
|
||||
env_vars: dict[str, str],
|
||||
distributed_init_method: str,
|
||||
driver_env_vars: dict[str, str] | None = None,
|
||||
assigned_physical_gpu_ids: list[int] | None = None,
|
||||
) -> None:
|
||||
@@ -163,6 +177,7 @@ class RayWorkerProc(WorkerProc):
|
||||
assigned_physical_gpu_ids
|
||||
)
|
||||
|
||||
self._init_kwargs["distributed_init_method"] = distributed_init_method
|
||||
self.local_rank = local_rank
|
||||
super().__init__(
|
||||
local_rank=local_rank,
|
||||
@@ -260,25 +275,6 @@ class RayExecutorV2(MultiprocExecutor):
|
||||
return {"num_gpus": num_devices}
|
||||
return {"num_gpus": 0, "resources": {device_key: num_devices}}
|
||||
|
||||
@staticmethod
|
||||
def _select_tcpstore_port(local_dp_rank: int | None, master_port: int) -> int:
|
||||
"""Pick the torch.distributed TCPStore port for this engine.
|
||||
|
||||
Co-located DP engines choosing this port with a shared random search
|
||||
collide intermittently. Seeding by node-local DP rank gives each a
|
||||
disjoint window. Non-DP engines and full windows fall back to a
|
||||
random port.
|
||||
"""
|
||||
if local_dp_rank is None:
|
||||
return get_open_port()
|
||||
# Offset past the DP master port reserved range, one window per rank.
|
||||
window = 32
|
||||
start_port = master_port + 100 + local_dp_rank * window
|
||||
try:
|
||||
return _get_open_port(start_port=start_port, max_attempts=window)
|
||||
except RuntimeError:
|
||||
return get_open_port()
|
||||
|
||||
def _init_executor(self) -> None:
|
||||
"""Initialize the RayExecutorV2 executor."""
|
||||
self._finalizer = weakref.finalize(self, self.shutdown)
|
||||
@@ -324,22 +320,7 @@ class RayExecutorV2(MultiprocExecutor):
|
||||
}
|
||||
)
|
||||
|
||||
# Step 3: Resolve the IP for torch.distributed TCPStore.
|
||||
# The TCPStore server runs on rank 0's node, so all workers
|
||||
# must be able to reach this address.
|
||||
dist_ip = bundle_assignments[0]["node_ip"]
|
||||
parallel_config = self.vllm_config.parallel_config
|
||||
local_dp_rank = (
|
||||
parallel_config.data_parallel_rank_local
|
||||
if parallel_config.data_parallel_size > 1
|
||||
else None
|
||||
)
|
||||
port = self._select_tcpstore_port(
|
||||
local_dp_rank, parallel_config.data_parallel_master_port
|
||||
)
|
||||
distributed_init_method = get_distributed_init_method(dist_ip, port)
|
||||
|
||||
# Step 4: Create broadcast MessageQueue.
|
||||
# Step 3: Create broadcast MessageQueue.
|
||||
# Workers on the driver node use shared memory; the rest use TCP.
|
||||
max_chunk_bytes = envs.VLLM_MQ_MAX_CHUNK_BYTES_MB * 1024 * 1024
|
||||
n_local = sum(1 for a in bundle_assignments if a["node_id"] == driver_node)
|
||||
@@ -351,9 +332,9 @@ class RayExecutorV2(MultiprocExecutor):
|
||||
)
|
||||
scheduler_output_handle = self.rpc_broadcast_mq.export_handle()
|
||||
|
||||
# Step 5: Spawn RayWorkerProc actors into PG bundles (deferred init).
|
||||
# Step 4: Spawn RayWorkerProc actors into PG bundles (deferred init).
|
||||
# Workers are created lightweight here; full initialization happens
|
||||
# in Step 7 after GPU IDs are discovered.
|
||||
# in Step 6 after GPU IDs are discovered.
|
||||
self.ray_worker_handles: list[RayWorkerHandle] = []
|
||||
instance_id = self.vllm_config.instance_id
|
||||
|
||||
@@ -405,7 +386,6 @@ class RayExecutorV2(MultiprocExecutor):
|
||||
.remote(
|
||||
vllm_config=self.vllm_config,
|
||||
rank=bundle["rank"],
|
||||
distributed_init_method=distributed_init_method,
|
||||
input_shm_handle=scheduler_output_handle,
|
||||
is_driver_worker=is_driver_worker,
|
||||
is_driver_node=is_driver_node,
|
||||
@@ -421,7 +401,7 @@ class RayExecutorV2(MultiprocExecutor):
|
||||
)
|
||||
self.ray_worker_handles.append(handle)
|
||||
|
||||
# Step 6: Discover physical GPU IDs assigned to each worker via Ray
|
||||
# Step 5: Discover physical GPU IDs assigned to each worker via Ray
|
||||
# runtime context.
|
||||
worker_node_and_physical_gpu_ids = ray.get(
|
||||
[
|
||||
@@ -440,8 +420,11 @@ class RayExecutorV2(MultiprocExecutor):
|
||||
for node_id, physical_gpu_ids in node_physical_gpu_ids.items():
|
||||
node_physical_gpu_ids[node_id] = sorted(physical_gpu_ids)
|
||||
|
||||
# Step 7: Initialize workers with local logical ranks and the
|
||||
# Step 6: Initialize workers with local logical ranks and the
|
||||
# logical-to-physical GPU mapping discovered from Ray placement.
|
||||
distributed_init_method = ray.get(
|
||||
self.ray_worker_handles[0].actor.create_dist_init_method.remote()
|
||||
)
|
||||
init_worker_refs = []
|
||||
for i, (node_id, _) in enumerate(worker_node_and_physical_gpu_ids):
|
||||
local_rank = node_workers[node_id].index(i)
|
||||
@@ -452,6 +435,7 @@ class RayExecutorV2(MultiprocExecutor):
|
||||
self.ray_worker_handles[i].actor.initialize_worker.remote(
|
||||
local_rank,
|
||||
worker_env_vars,
|
||||
distributed_init_method,
|
||||
self.driver_env_vars,
|
||||
assigned_physical_gpu_ids=assigned_physical_gpu_ids,
|
||||
)
|
||||
@@ -465,7 +449,7 @@ class RayExecutorV2(MultiprocExecutor):
|
||||
)
|
||||
ray.get(init_worker_refs)
|
||||
|
||||
# Step 8: Collect response MQ handles
|
||||
# Step 7: Collect response MQ handles
|
||||
init_results = ray.get(
|
||||
[h.actor.wait_for_init.remote() for h in self.ray_worker_handles]
|
||||
)
|
||||
@@ -478,12 +462,12 @@ class RayExecutorV2(MultiprocExecutor):
|
||||
MessageQueue.create_from_handle(result["handle"], 0)
|
||||
)
|
||||
|
||||
# Step 9: Start run() before wait_until_ready() to avoid
|
||||
# Step 8: Start run() before wait_until_ready() to avoid
|
||||
# deadlock — workers send subscriptions inside run().
|
||||
for handle in self.ray_worker_handles:
|
||||
handle.run()
|
||||
|
||||
# Step 10: wait_until_ready() barrier
|
||||
# Step 9: wait_until_ready() barrier
|
||||
self.rpc_broadcast_mq.wait_until_ready()
|
||||
for response_mq in self.response_mqs:
|
||||
response_mq.wait_until_ready()
|
||||
|
||||
Reference in New Issue
Block a user