diff --git a/tests/v1/engine/test_core_engine_actor_manager.py b/tests/v1/engine/test_core_engine_actor_manager.py index 69f52e2a7f..f1a67ad9ba 100644 --- a/tests/v1/engine/test_core_engine_actor_manager.py +++ b/tests/v1/engine/test_core_engine_actor_manager.py @@ -20,6 +20,8 @@ from vllm.v1.engine.core_client import BackgroundResources from vllm.v1.engine.utils import ( CoreEngineActorManager, EngineZmqAddresses, + _dp_nodes_master_first, + _node_ip_from_resources, get_engine_zmq_addresses, launch_core_engines, ) @@ -363,3 +365,110 @@ def test_ray_dp_addresses_resolved_before_actor_creation( "time they DEALER-connect. See PR #42585 / Ray-DP " "multi-API-server regression." ) + + +def test_dp_nodes_master_first_orders_and_validates(): + master = {"GPU": 2.0, "node:10.0.0.1": 1.0, "node:__internal_head__": 1.0} + worker = {"GPU": 2.0, "node:10.0.0.2": 1.0} + + nodes = _dp_nodes_master_first({"w": worker, "m": master}, "10.0.0.1") + assert [node_id for node_id, _ in nodes] == ["m", "w"] + + with pytest.raises(AssertionError, match="DP master node"): + _dp_nodes_master_first({"w": worker}, "10.0.0.1") + with pytest.raises(AssertionError, match="one head node"): + _dp_nodes_master_first({"a": master, "b": dict(master)}, "10.0.0.1") + + +@pytest.fixture +def ray_2gpu_node(monkeypatch: pytest.MonkeyPatch): + """Real single-node Ray with 2 virtual GPUs and no dashboard. + + Resource maps and placement groups are real. Without the dashboard any + ``ray.util.state.list_nodes()`` call fails, the production condition this + path must survive. Only the platform device key is patched ("" on CPU). + """ + from ray._private.state import available_resources_per_node + + if ray.is_initialized(): + ray.shutdown() + ray.init(num_cpus=4, num_gpus=2, include_dashboard=False, log_to_driver=False) + monkeypatch.setattr("vllm.v1.engine.utils.current_platform.ray_device_key", "GPU") + + (node_resources,) = available_resources_per_node().values() + master_ip = _node_ip_from_resources(node_resources) + assert master_ip is not None + created = [] + + def hold_gpus(num_gpus: int): + """Occupy GPUs so the node looks like it already runs engines.""" + pg = ray.util.placement_group([{"GPU": 1.0}] * num_gpus) + ray.get(pg.ready(), timeout=60) + created.append(pg) + + yield SimpleNamespace(master_ip=master_ip, hold_gpus=hold_gpus, created=created) + + for pg in created: + ray.util.remove_placement_group(pg) + ray.shutdown() + + +def _elastic_ep_config(dp_size: int, world_size: int, master_ip: str): + return SimpleNamespace( + parallel_config=SimpleNamespace( + data_parallel_size=dp_size, + data_parallel_master_ip=master_ip, + world_size=world_size, + ) + ) + + +@pytest.mark.timeout(120) +def test_add_dp_placement_groups_does_not_require_ray_default(ray_2gpu_node): + """DP 1 -> 2 on an idle node: one schedulable group pinned to the master, + found without the dashboard (``list_nodes`` would fail here).""" + ip = ray_2gpu_node.master_ip + + pgs, local_ranks = CoreEngineActorManager.add_dp_placement_groups( + _elastic_ep_config(dp_size=1, world_size=1, master_ip=ip), 2 + ) + ray_2gpu_node.created.extend(pgs) + + assert [pg.bundle_specs for pg in pgs] == [ + [{"GPU": 1.0, f"node:{ip}": 0.001}, {"CPU": 1.0}] + ] + assert local_ranks == [0] + ray.get(pgs[0].ready(), timeout=60) + + +@pytest.mark.timeout(120) +def test_add_dp_placement_groups_counts_engines_already_on_node(ray_2gpu_node): + """One GPU already held -> the new engine gets local rank 1.""" + ray_2gpu_node.hold_gpus(1) + + pgs, local_ranks = CoreEngineActorManager.add_dp_placement_groups( + _elastic_ep_config(dp_size=1, world_size=1, master_ip=ray_2gpu_node.master_ip), + 2, + ) + ray_2gpu_node.created.extend(pgs) + + assert len(pgs) == 1 + assert local_ranks == [1] + ray.get(pgs[0].ready(), timeout=60) + + +@pytest.mark.timeout(120) +def test_add_dp_placement_groups_respects_world_size(ray_2gpu_node): + """With TP=2 a node with one idle GPU cannot host a new engine.""" + ray_2gpu_node.hold_gpus(1) + + assert CoreEngineActorManager.add_dp_placement_groups( + _elastic_ep_config(dp_size=1, world_size=2, master_ip=ray_2gpu_node.master_ip), + 2, + ) == ([], []) + + +def test_add_dp_placement_groups_noop_without_growth(): + assert CoreEngineActorManager.add_dp_placement_groups( + _elastic_ep_config(dp_size=2, world_size=1, master_ip="10.0.0.1"), 2 + ) == ([], []) diff --git a/vllm/v1/engine/utils.py b/vllm/v1/engine/utils.py index 02b49819fd..e786cf2c5b 100644 --- a/vllm/v1/engine/utils.py +++ b/vllm/v1/engine/utils.py @@ -124,6 +124,31 @@ def _get_bundle_node_ip(bundle: dict[str, float]) -> str: raise ValueError(f"Missing node affinity in placement bundle: {bundle}") +def _dp_nodes_master_first( + available_resources: dict[str, dict], dp_master_ip: str +) -> list[tuple[str, dict]]: + """Live Ray nodes as ``(node_id, resources)`` with the DP master first. + + The master is found through the ``node:`` resource Ray sets on every + node, so this works with the resource maps from ``ray._private.state``. + Do not replace it with ``ray.util.state.list_nodes()``: that API goes + through the dashboard HTTP server and only exists with ``ray[default]`` + (see #23822). + """ + master_key = f"node:{dp_master_ip}" + nodes = sorted( + available_resources.items(), key=lambda item: master_key not in item[1] + ) + assert len(nodes) > 0, "No nodes with resources found in Ray cluster." + assert master_key in nodes[0][1], ( + f"The DP master node (ip: {dp_master_ip}) is missing or dead" + ) + assert len(nodes) == 1 or master_key not in nodes[1][1], ( + "There can only be one head node" + ) + return nodes + + def _node_ip_from_resources(node_resources: dict) -> str | None: """Return the node IP encoded in a Ray per-node resource dict, or None. @@ -561,14 +586,12 @@ class CoreEngineActorManager: placement_groups: list[PlacementGroup] = [] local_dp_ranks: list[int] = [] - dp_master_ip_key = f"node:{dp_master_ip}" - nodes = sorted( - available_resources.values(), key=lambda x: dp_master_ip_key not in x - ) - assert len(nodes) > 0, "No nodes with resources found in Ray cluster." - assert dp_master_ip_key in nodes[0], ( - f"The DP master node (ip: {dp_master_ip}) is missing or dead" - ) + nodes = [ + node_resources + for _, node_resources in _dp_nodes_master_first( + available_resources, dp_master_ip + ) + ] # optionally restrict DP placement to a caller-provided node set. requested_node_ips = { @@ -767,7 +790,6 @@ class CoreEngineActorManager: available_resources_per_node, total_resources_per_node, ) - from ray.util.state import list_nodes old_dp_size = old_vllm_config.parallel_config.data_parallel_size num_pg_to_create = new_data_parallel_size - old_dp_size @@ -778,30 +800,27 @@ class CoreEngineActorManager: dp_master_ip = old_vllm_config.parallel_config.data_parallel_master_ip world_size = old_vllm_config.parallel_config.world_size - nodes = list_nodes() - nodes = sorted(nodes, key=lambda node: node.node_ip != dp_master_ip) - assert nodes[0].node_ip == dp_master_ip, "The first node must be the head node" - assert len(nodes) == 1 or nodes[1].node_ip != dp_master_ip, ( - "There can only be one head node" - ) - + # Both maps are keyed by node id and only contain live nodes. available_resources = available_resources_per_node() total_resources = total_resources_per_node() + nodes = _dp_nodes_master_first(available_resources, dp_master_ip) placement_groups = [] local_dp_ranks = [] num_pg_created = 0 device_str = current_platform.ray_device_key - for node in nodes: + for node_id, node_resources in nodes: if num_pg_created >= num_pg_to_create: break - node_ip = node.node_ip - node_id = node.node_id - if device_str not in available_resources[node_id]: + node_ip = _node_ip_from_resources(node_resources) + assert node_ip is not None, ( + f"No node IP key found in node resources: {node_resources}" + ) + if device_str not in node_resources: continue - available_gpus = int(available_resources[node_id][device_str]) + available_gpus = int(node_resources[device_str]) # Get total GPUs on this node from the node's resources # Ray stores node resources with node ID as key