[Feat][Executor] Introduce RayExecutorV2 (#36836)

Signed-off-by: Jeffrey Wang <jeffreywang@anyscale.com>
This commit is contained in:
Jeffrey Wang
2026-04-01 14:34:29 -07:00
committed by GitHub
parent cb268e4e55
commit de5e6c44c6
14 changed files with 1603 additions and 30 deletions
+155 -14
View File
@@ -26,6 +26,17 @@ if TYPE_CHECKING:
logger = init_logger(__name__)
PG_WAIT_TIMEOUT = 1800
# Env vars that are worker-specific and must NOT be copied from the
# driver to Ray workers — they are set per-worker after GPU discovery.
WORKER_SPECIFIC_ENV_VARS: set[str] = {
"VLLM_HOST_IP",
"VLLM_HOST_PORT",
"LOCAL_RANK",
"CUDA_VISIBLE_DEVICES",
"HIP_VISIBLE_DEVICES",
"ROCR_VISIBLE_DEVICES",
}
try:
import ray
from ray.util import placement_group_table
@@ -51,6 +62,8 @@ try:
# that thread.
self.compiled_dag_cuda_device_set = False
rpc_rank: int
def adjust_rank(self, rank_mapping: dict[int, int]) -> None:
"""
Adjust the rpc_rank based on the given mapping.
@@ -214,13 +227,17 @@ def assert_ray_available():
def _verify_bundles(
placement_group: "PlacementGroup", parallel_config: ParallelConfig, device_str: str
placement_group: "PlacementGroup",
parallel_config: ParallelConfig,
device_str: str,
require_gpu_on_driver: bool = True,
):
"""Verify a given placement group has bundles located in the right place.
There are 2 rules.
- Warn if all tensor parallel workers cannot fit in a single node.
- Fail if driver node is not included in a placement group.
- Fail if driver node is not included in a placement group
(only when require_gpu_on_driver is True).
"""
assert ray.is_initialized(), (
"Ray is not initialized although distributed-executor-backend is ray."
@@ -237,7 +254,7 @@ def _verify_bundles(
node_id_to_bundle[node_id].append(bundles[bundle_idx])
driver_node_id = ray.get_runtime_context().get_node_id()
if driver_node_id not in node_id_to_bundle:
if require_gpu_on_driver and driver_node_id not in node_id_to_bundle:
raise RuntimeError(
f"driver node id {driver_node_id} is not included in a placement "
f"group {placement_group.id}. Node id -> bundles "
@@ -266,6 +283,115 @@ def _verify_bundles(
)
def build_actor_name(
instance_id: str,
rank: int,
tp_size: int,
pp_size: int,
pcp_size: int,
) -> str:
"""Build a descriptive Ray actor name for dashboard visibility."""
name = f"vllm_Worker_{instance_id}"
if tp_size > 1:
name += f"_TP{rank % tp_size}"
if pp_size > 1:
name += f"_PP{(rank // tp_size) % pp_size}"
if pcp_size > 1:
name += f"_PCP{rank // (tp_size * pp_size)}"
return name
def get_bundles_for_indices(
placement_group: "PlacementGroup",
bundle_indices: list[int],
world_size: int,
) -> list[tuple[int, str, str]]:
"""
Return GPU bundle indices paired with node IDs and node IPs for
explicit bundle indices specified via VLLM_RAY_BUNDLE_INDICES.
"""
assert len(bundle_indices) == world_size, (
"VLLM_RAY_BUNDLE_INDICES must have the same size"
f" as the world size, but got {bundle_indices=} "
f"and {world_size=}"
)
assert len(set(bundle_indices)) == len(bundle_indices), (
"VLLM_RAY_BUNDLE_INDICES cannot have duplicate values,"
f" but got {bundle_indices=}"
)
pg_data = placement_group_table(placement_group)
pg_bundle_to_node = pg_data["bundles_to_node_id"]
node_id_to_ip = {
n["NodeID"]: n["NodeManagerAddress"] for n in ray.nodes() if n["Alive"]
}
return [
(bid, pg_bundle_to_node[bid], node_id_to_ip[pg_bundle_to_node[bid]])
for bid in bundle_indices
]
def get_bundles_sorted_by_node(
placement_group: "PlacementGroup",
) -> list[tuple[int, str, str]]:
"""
Return GPU bundle indices paired with node IDs and node IPs,
sorted driver-first.
This utility has to be invoked from the driver node.
Example: 3-node cluster, driver on node-A, PG bundles spread
across nodes:
Input: [
(0, node-C),
(1, node-A),
(2, node-B),
(3, node-C),
(4, node-A),
(5, node-B),
]
Output: [
(1, node-A),
(4, node-A),
(2, node-B),
(5, node-B),
(0, node-C),
(3, node-C),
]
"""
pg_data = placement_group_table(placement_group)
bundle_to_node = pg_data["bundles_to_node_id"]
ray_device_key = current_platform.ray_device_key
if not ray_device_key:
raise ValueError(
f"current platform {current_platform.device_name} does not support ray."
)
node_id_to_ip = {
n["NodeID"]: n["NodeManagerAddress"] for n in ray.nodes() if n["Alive"]
}
bundle_specs = placement_group.bundle_specs
assert bundle_specs is not None
bundle_to_node_id: list[tuple[int, str, str]] = []
for bundle_idx, bundle in enumerate(bundle_specs):
if bundle.get(ray_device_key):
node_id = bundle_to_node.get(bundle_idx)
bundle_to_node_id.append((bundle_idx, node_id, node_id_to_ip[node_id]))
driver_node = ray.get_runtime_context().get_node_id()
def _sort_key(item):
_, node_id, _ = item
return (0 if node_id == driver_node else 1, node_id)
bundle_to_node_id.sort(key=_sort_key)
return bundle_to_node_id
def _wait_until_pg_ready(current_placement_group: "PlacementGroup"):
"""Wait until a placement group is ready.
@@ -352,6 +478,7 @@ def _wait_until_pg_removed(current_placement_group: "PlacementGroup"):
def initialize_ray_cluster(
parallel_config: ParallelConfig,
ray_address: str | None = None,
require_gpu_on_driver: bool = True,
):
"""Initialize the distributed cluster with Ray.
@@ -363,10 +490,18 @@ def initialize_ray_cluster(
parallel_config: The configurations for parallel execution.
ray_address: The address of the Ray cluster. If None, uses
the default Ray cluster address.
require_gpu_on_driver: If True (default), require at least one GPU
on the current (driver) node and pin the first PG bundle to it.
Set to False for executors like RayExecutorV2 where all GPU work
is delegated to remote Ray actors.
"""
assert_ray_available()
from vllm.platforms import current_platform
# Disable Ray usage stats collection
if os.environ.get("RAY_USAGE_STATS_ENABLED", "0") != "1":
os.environ["RAY_USAGE_STATS_ENABLED"] = "0"
# Prevalidate GPU requirements before Ray processing
if current_platform.is_cuda() and parallel_config.world_size > 1:
available_gpus = current_platform.device_count()
@@ -459,16 +594,20 @@ def initialize_ray_cluster(
current_ip = get_ip()
current_node_id = ray.get_runtime_context().get_node_id()
current_node_resource = available_resources_per_node()[current_node_id]
if current_node_resource.get(device_str, 0) < 1:
raise ValueError(
f"Current node has no {device_str} available. "
f"{current_node_resource=}. vLLM engine cannot start without "
f"{device_str}. Make sure you have at least 1 {device_str} "
f"available in a node {current_node_id=} {current_ip=}."
)
# This way, at least bundle is required to be created in a current
# node.
placement_group_specs[0][f"node:{current_ip}"] = 0.001
# TODO (jeffreywang): require_gpu_on_driver should be always False
# after deprecating RayDistributedExecutor.
if require_gpu_on_driver:
if current_node_resource.get(device_str, 0) < 1:
raise ValueError(
f"Current node has no {device_str} available. "
f"{current_node_resource=}. vLLM engine cannot start "
f"without {device_str}. Make sure you have at least 1 "
f"{device_str} available in a node "
f"{current_node_id=} {current_ip=}."
)
# This way, at least bundle is required to be created in a
# current node.
placement_group_specs[0][f"node:{current_ip}"] = 0.001
# By default, Ray packs resources as much as possible.
current_placement_group = ray.util.placement_group(
@@ -477,7 +616,9 @@ def initialize_ray_cluster(
_wait_until_pg_ready(current_placement_group)
assert current_placement_group is not None
_verify_bundles(current_placement_group, parallel_config, device_str)
_verify_bundles(
current_placement_group, parallel_config, device_str, require_gpu_on_driver
)
# Set the placement group in the parallel config
parallel_config.placement_group = current_placement_group