[Feat][Executor] Introduce RayExecutorV2 (#36836)
Signed-off-by: Jeffrey Wang <jeffreywang@anyscale.com>
This commit is contained in:
@@ -7,6 +7,7 @@ from concurrent.futures import Future
|
||||
from functools import cached_property
|
||||
from typing import TYPE_CHECKING, Literal, TypeVar, overload
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed.kv_transfer.kv_connector.utils import KVOutputAggregator
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.base import (
|
||||
@@ -57,9 +58,14 @@ class Executor(ABC):
|
||||
)
|
||||
executor_class = distributed_executor_backend
|
||||
elif distributed_executor_backend == "ray":
|
||||
from vllm.v1.executor.ray_executor import RayDistributedExecutor
|
||||
if envs.VLLM_USE_RAY_V2_EXECUTOR_BACKEND:
|
||||
from vllm.v1.executor.ray_executor_v2 import RayExecutorV2
|
||||
|
||||
executor_class = RayDistributedExecutor
|
||||
executor_class = RayExecutorV2
|
||||
else:
|
||||
from vllm.v1.executor.ray_executor import RayDistributedExecutor
|
||||
|
||||
executor_class = RayDistributedExecutor
|
||||
elif distributed_executor_backend == "mp":
|
||||
from vllm.v1.executor.multiproc_executor import MultiprocExecutor
|
||||
|
||||
|
||||
@@ -0,0 +1,18 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import os
|
||||
|
||||
from vllm.ray.ray_env import RAY_NON_CARRY_OVER_ENV_VARS
|
||||
|
||||
|
||||
def get_driver_env_vars(
|
||||
worker_specific_vars: set[str],
|
||||
) -> dict[str, str]:
|
||||
"""Return driver env vars to propagate to Ray workers.
|
||||
|
||||
Returns everything from ``os.environ`` except ``worker_specific_vars``
|
||||
and user-configured exclusions (``RAY_NON_CARRY_OVER_ENV_VARS``).
|
||||
"""
|
||||
exclude_vars = worker_specific_vars | RAY_NON_CARRY_OVER_ENV_VARS
|
||||
|
||||
return {key: value for key, value in os.environ.items() if key not in exclude_vars}
|
||||
@@ -23,6 +23,7 @@ from vllm.v1.core.sched.output import GrammarOutput, SchedulerOutput
|
||||
from vllm.v1.engine import ReconfigureDistributedRequest, ReconfigureRankType
|
||||
from vllm.v1.executor.abstract import Executor
|
||||
from vllm.v1.executor.ray_utils import (
|
||||
WORKER_SPECIFIC_ENV_VARS,
|
||||
FutureWrapper,
|
||||
RayWorkerWrapper,
|
||||
initialize_ray_cluster,
|
||||
@@ -62,17 +63,6 @@ class RayWorkerMetaData:
|
||||
class RayDistributedExecutor(Executor):
|
||||
"""Ray-based distributed executor"""
|
||||
|
||||
# These env vars are worker-specific, therefore are NOT copied
|
||||
# from the driver to the workers
|
||||
WORKER_SPECIFIC_ENV_VARS = {
|
||||
"VLLM_HOST_IP",
|
||||
"VLLM_HOST_PORT",
|
||||
"LOCAL_RANK",
|
||||
"CUDA_VISIBLE_DEVICES",
|
||||
"HIP_VISIBLE_DEVICES",
|
||||
"ROCR_VISIBLE_DEVICES",
|
||||
}
|
||||
|
||||
uses_ray: bool = True
|
||||
supports_pp: bool = True
|
||||
|
||||
@@ -335,7 +325,7 @@ class RayDistributedExecutor(Executor):
|
||||
|
||||
# Environment variables to copy from driver to workers
|
||||
env_vars_to_copy = get_env_vars_to_copy(
|
||||
exclude_vars=self.WORKER_SPECIFIC_ENV_VARS,
|
||||
exclude_vars=WORKER_SPECIFIC_ENV_VARS,
|
||||
additional_vars=set(current_platform.additional_env_vars),
|
||||
destination="workers",
|
||||
)
|
||||
|
||||
@@ -0,0 +1,524 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import copy
|
||||
import os
|
||||
import threading
|
||||
import weakref
|
||||
from collections import defaultdict, deque
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed.device_communicators.shm_broadcast import (
|
||||
Handle,
|
||||
MessageQueue,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.network_utils import (
|
||||
get_distributed_init_method,
|
||||
get_open_port,
|
||||
)
|
||||
from vllm.v1.executor.multiproc_executor import (
|
||||
FutureWrapper,
|
||||
MultiprocExecutor,
|
||||
WorkerProc,
|
||||
)
|
||||
from vllm.v1.executor.ray_env_utils import get_driver_env_vars
|
||||
from vllm.v1.executor.ray_utils import (
|
||||
WORKER_SPECIFIC_ENV_VARS,
|
||||
build_actor_name,
|
||||
get_bundles_for_indices,
|
||||
get_bundles_sorted_by_node,
|
||||
initialize_ray_cluster,
|
||||
ray,
|
||||
)
|
||||
|
||||
if ray is not None:
|
||||
from ray.actor import ActorHandle
|
||||
from ray.types import ObjectRef
|
||||
from ray.util.scheduling_strategies import PlacementGroupSchedulingStrategy
|
||||
else:
|
||||
ActorHandle = None
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
@dataclass
|
||||
class RayWorkerHandle:
|
||||
"""Handle for a Ray worker actor, compatible with MultiprocExecutor."""
|
||||
|
||||
actor: ActorHandle
|
||||
"""Ray worker actor"""
|
||||
|
||||
rank: int
|
||||
"""Rank of the worker"""
|
||||
|
||||
local_rank: int
|
||||
"""Local rank of the worker"""
|
||||
|
||||
node_id: str
|
||||
"""Node ID of the worker"""
|
||||
|
||||
bundle_id_idx: int = -1
|
||||
"""Placement group bundle index for the worker"""
|
||||
|
||||
run_ref: ObjectRef | None = None
|
||||
"""run() ObjectRef used as a sentinel for health monitoring"""
|
||||
|
||||
def run(self):
|
||||
"""Start the worker's busy loop"""
|
||||
self.run_ref = self.actor.run.remote()
|
||||
|
||||
|
||||
class RayWorkerProc(WorkerProc):
|
||||
"""Worker process that runs inside a Ray actor.
|
||||
|
||||
Initialization is split into two phases:
|
||||
1. __init__: lightweight setup, stores init args (no device/model init)
|
||||
2. initialize_worker: called after GPU IDs are discovered, completes
|
||||
the full WorkerProc initialization with the correct local_rank and
|
||||
CUDA_VISIBLE_DEVICES.
|
||||
|
||||
CUDA_VISIBLE_DEVICES setup flow:
|
||||
|
||||
1. RayExecutorV2 enables RAY_EXPERIMENTAL_NOSET_CUDA_VISIBLE_DEVICES so Ray does
|
||||
not set CUDA_VISIBLE_DEVICES on RayWorkerProc actors at creation time.
|
||||
2. Each actor is scheduled with a placement group and bundle index; Ray resolves
|
||||
the physical GPU ID for that bundle at placement time.
|
||||
3. After placement, the worker discovers that GPU ID and sets
|
||||
CUDA_VISIBLE_DEVICES before finishing WorkerProc initialization.
|
||||
|
||||
There is no workaround for this unset-and-reset sequence when the placement group
|
||||
is externally managed: scheduling must complete before CUDA_VISIBLE_DEVICES can
|
||||
match the GPU tied to the worker's bundle.
|
||||
|
||||
This sequence allows multiple vLLM instances to coexist on the same node:
|
||||
each instance is unaware which physical devices others hold, and the
|
||||
externally managed placement group avoids CUDA_VISIBLE_DEVICES conflicts
|
||||
by binding workers to specific placement group bundles.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vllm_config: VllmConfig,
|
||||
rank: int,
|
||||
distributed_init_method: str,
|
||||
input_shm_handle: Handle,
|
||||
is_driver_worker: bool,
|
||||
is_driver_node: bool = False,
|
||||
):
|
||||
# Defer WorkerProc.__init__ until GPU IDs are known.
|
||||
self._is_driver_node = is_driver_node
|
||||
self._init_kwargs = dict(
|
||||
vllm_config=vllm_config,
|
||||
rank=rank,
|
||||
distributed_init_method=distributed_init_method,
|
||||
input_shm_handle=input_shm_handle,
|
||||
shared_worker_lock=None,
|
||||
is_driver_worker=is_driver_worker,
|
||||
)
|
||||
|
||||
def get_node_and_gpu_ids(self) -> tuple[str, list[int]]:
|
||||
"""Return (node_id, gpu_ids) assigned to this actor by Ray."""
|
||||
node_id = ray.get_runtime_context().get_node_id()
|
||||
device_key = current_platform.ray_device_key
|
||||
if not device_key:
|
||||
raise RuntimeError(
|
||||
f"current platform {current_platform.device_name} does not support ray."
|
||||
)
|
||||
gpu_ids = ray.get_runtime_context().get_accelerator_ids()[device_key]
|
||||
return node_id, [int(x) for x in gpu_ids]
|
||||
|
||||
def initialize_worker(
|
||||
self,
|
||||
local_rank: int,
|
||||
env_vars: dict[str, str],
|
||||
driver_env_vars: dict[str, str] | None = None,
|
||||
) -> None:
|
||||
"""Complete initialization after GPU assignment is known.
|
||||
|
||||
*driver_env_vars* are applied with ``setdefault`` — they fill
|
||||
in missing vars but never overwrite node-local values.
|
||||
*env_vars* (e.g. CUDA_VISIBLE_DEVICES) always overwrite.
|
||||
"""
|
||||
if driver_env_vars:
|
||||
for key, value in driver_env_vars.items():
|
||||
os.environ.setdefault(key, value)
|
||||
for key, value in env_vars.items():
|
||||
os.environ[key] = value
|
||||
|
||||
self.local_rank = local_rank
|
||||
super().__init__(
|
||||
local_rank=local_rank,
|
||||
**self._init_kwargs,
|
||||
)
|
||||
|
||||
def _init_message_queues(
|
||||
self, input_shm_handle: Handle, vllm_config: VllmConfig
|
||||
) -> None:
|
||||
"""
|
||||
Workers on the same node as the executor use shared memory for
|
||||
both the broadcast (input) MQ and the response MQ. Workers on
|
||||
different nodes use TCP (n_local_reader=0).
|
||||
"""
|
||||
self.rpc_broadcast_mq = MessageQueue.create_from_handle(
|
||||
input_shm_handle, self.worker.rank
|
||||
)
|
||||
|
||||
n_local = 1 if self._is_driver_node else 0
|
||||
# Use ray.util.get_node_ip_address() to get Ray's internal IP.
|
||||
# get_ip() returns host's external IP which is typically not
|
||||
# routable between nodes within the cluster.
|
||||
self.worker_response_mq = MessageQueue(
|
||||
n_reader=1,
|
||||
n_local_reader=n_local,
|
||||
connect_ip=ray.util.get_node_ip_address(),
|
||||
)
|
||||
self.peer_response_handles: list[dict] = []
|
||||
|
||||
def wait_for_init(self) -> dict:
|
||||
"""Respond to the driver's wait_until_ready() barrier."""
|
||||
assert self.worker_response_mq is not None
|
||||
return {
|
||||
"status": self.READY_STR,
|
||||
"handle": self.worker_response_mq.export_handle(),
|
||||
}
|
||||
|
||||
def run(self) -> None:
|
||||
"""Main entry point called via actor.run.remote()."""
|
||||
try:
|
||||
assert self.rpc_broadcast_mq is not None
|
||||
self.rpc_broadcast_mq.wait_until_ready()
|
||||
assert self.worker_response_mq is not None
|
||||
self.worker_response_mq.wait_until_ready()
|
||||
|
||||
self.worker_busy_loop()
|
||||
except Exception as e:
|
||||
logger.exception("RayWorkerProc failed: %s", e)
|
||||
raise
|
||||
finally:
|
||||
self.shutdown()
|
||||
|
||||
|
||||
class RayExecutorV2(MultiprocExecutor):
|
||||
"""Ray-based distributed executor using MessageQueue communication.
|
||||
|
||||
Inherits from MultiprocExecutor to reuse the MQ-based control plane
|
||||
and NCCL data plane. Workers are Ray actors.
|
||||
|
||||
Async scheduling is enabled, inherited from MultiprocExecutor.
|
||||
This is cricitcal for RayExecutorV2 to be performant.
|
||||
"""
|
||||
|
||||
uses_ray: bool = True
|
||||
supports_pp: bool = True
|
||||
|
||||
def __init__(self, vllm_config: VllmConfig):
|
||||
super().__init__(vllm_config)
|
||||
|
||||
def _build_runtime_env(self) -> dict:
|
||||
"""Build a runtime_env dict for RayWorkerProc actors.
|
||||
|
||||
Driver env vars are applied separately via initialize_worker
|
||||
with setdefault semantics.
|
||||
"""
|
||||
base = self.parallel_config.ray_runtime_env
|
||||
runtime_env: dict = copy.deepcopy(dict(base)) if base else {}
|
||||
|
||||
env_vars = runtime_env.setdefault("env_vars", {})
|
||||
env_vars.update({v: "1" for v in current_platform.ray_noset_device_env_vars})
|
||||
if self.parallel_config.ray_workers_use_nsight:
|
||||
runtime_env["nsight"] = {
|
||||
"t": "cuda,cudnn,cublas",
|
||||
"o": "'worker_process_%p'",
|
||||
"cuda-graph-trace": "node",
|
||||
}
|
||||
return runtime_env
|
||||
|
||||
@staticmethod
|
||||
def _get_actor_resource_kwargs() -> dict[str, Any]:
|
||||
"""Return Ray actor resource kwargs for the current platform."""
|
||||
num_devices = envs.VLLM_RAY_PER_WORKER_GPUS
|
||||
device_key = current_platform.ray_device_key
|
||||
if device_key == "GPU":
|
||||
return {"num_gpus": num_devices}
|
||||
return {"num_gpus": 0, "resources": {device_key: num_devices}}
|
||||
|
||||
def _init_executor(self) -> None:
|
||||
"""Initialize the RayExecutorV2 executor."""
|
||||
self._finalizer = weakref.finalize(self, self.shutdown)
|
||||
self.is_failed = False
|
||||
self.failure_callback = None
|
||||
self.shutting_down = False
|
||||
self.shutdown_lock = threading.Lock()
|
||||
|
||||
# Step 1: Initialize Ray cluster and retrieve placement group
|
||||
if ray is None:
|
||||
raise ImportError("Using Ray backend requires installation of ray.")
|
||||
initialize_ray_cluster(self.parallel_config, require_gpu_on_driver=False)
|
||||
placement_group = self.parallel_config.placement_group
|
||||
|
||||
tp_size, pp_size, pcp_size = self._get_parallel_sizes()
|
||||
assert self.world_size == tp_size * pp_size * pcp_size, (
|
||||
f"world_size ({self.world_size}) must be equal to the "
|
||||
f"tensor_parallel_size ({tp_size}) x pipeline"
|
||||
f"_parallel_size ({pp_size}) x prefill_context"
|
||||
f"_parallel_size ({pcp_size}). "
|
||||
)
|
||||
|
||||
# Step 2: Build bundle assignments for worker rank placement
|
||||
# while respecting VLLM_RAY_BUNDLE_INDICES.
|
||||
if envs.VLLM_RAY_BUNDLE_INDICES:
|
||||
bundle_to_node_id = get_bundles_for_indices(
|
||||
placement_group,
|
||||
list(map(int, envs.VLLM_RAY_BUNDLE_INDICES.split(","))),
|
||||
self.world_size,
|
||||
)
|
||||
else:
|
||||
bundle_to_node_id = get_bundles_sorted_by_node(placement_group)
|
||||
driver_node = ray.get_runtime_context().get_node_id()
|
||||
|
||||
bundle_assignments: list[dict[str, Any]] = []
|
||||
for rank, (bundle_id_idx, node_id, node_ip) in enumerate(bundle_to_node_id):
|
||||
bundle_assignments.append(
|
||||
{
|
||||
"rank": rank,
|
||||
"bundle_id_idx": bundle_id_idx,
|
||||
"node_id": node_id,
|
||||
"node_ip": node_ip,
|
||||
}
|
||||
)
|
||||
|
||||
# Step 3: Resolve the IP for torch.distributed TCPStore.
|
||||
# The TCPStore server runs on rank 0's node, so all workers
|
||||
# must be able to reach this address.
|
||||
dist_ip = bundle_assignments[0]["node_ip"]
|
||||
distributed_init_method = get_distributed_init_method(dist_ip, get_open_port())
|
||||
|
||||
# Step 4: Create broadcast MessageQueue.
|
||||
# Workers on the driver node use shared memory; the rest use TCP.
|
||||
max_chunk_bytes = envs.VLLM_MQ_MAX_CHUNK_BYTES_MB * 1024 * 1024
|
||||
n_local = sum(1 for a in bundle_assignments if a["node_id"] == driver_node)
|
||||
self.rpc_broadcast_mq = MessageQueue(
|
||||
self.world_size,
|
||||
n_local,
|
||||
max_chunk_bytes=max_chunk_bytes,
|
||||
connect_ip=ray.util.get_node_ip_address(),
|
||||
)
|
||||
scheduler_output_handle = self.rpc_broadcast_mq.export_handle()
|
||||
|
||||
# Step 5: Spawn RayWorkerProc actors into PG bundles (deferred init).
|
||||
# Workers are created lightweight here; full initialization happens
|
||||
# in Step 7 after GPU IDs are discovered.
|
||||
self.ray_worker_handles: list[RayWorkerHandle] = []
|
||||
instance_id = self.vllm_config.instance_id
|
||||
|
||||
# Collect driver env vars and apply but don't overwrite node-local values.
|
||||
self.driver_env_vars = get_driver_env_vars(
|
||||
worker_specific_vars=WORKER_SPECIFIC_ENV_VARS,
|
||||
)
|
||||
|
||||
runtime_env = self._build_runtime_env()
|
||||
resource_kwargs = self._get_actor_resource_kwargs()
|
||||
|
||||
for bundle_idx in range(self.world_size):
|
||||
bundle = bundle_assignments[bundle_idx]
|
||||
is_driver_worker = self._is_driver_worker(bundle["rank"])
|
||||
is_driver_node = bundle["node_id"] == driver_node
|
||||
|
||||
scheduling_strategy = PlacementGroupSchedulingStrategy(
|
||||
placement_group=placement_group,
|
||||
placement_group_bundle_index=bundle["bundle_id_idx"],
|
||||
)
|
||||
|
||||
actor_name = build_actor_name(
|
||||
instance_id, bundle["rank"], tp_size, pp_size, pcp_size
|
||||
)
|
||||
|
||||
actor = (
|
||||
ray.remote(RayWorkerProc)
|
||||
.options(
|
||||
name=actor_name,
|
||||
num_cpus=0,
|
||||
**resource_kwargs,
|
||||
scheduling_strategy=scheduling_strategy,
|
||||
runtime_env=runtime_env,
|
||||
)
|
||||
.remote(
|
||||
vllm_config=self.vllm_config,
|
||||
rank=bundle["rank"],
|
||||
distributed_init_method=distributed_init_method,
|
||||
input_shm_handle=scheduler_output_handle,
|
||||
is_driver_worker=is_driver_worker,
|
||||
is_driver_node=is_driver_node,
|
||||
)
|
||||
)
|
||||
|
||||
handle = RayWorkerHandle(
|
||||
actor=actor,
|
||||
rank=bundle["rank"],
|
||||
local_rank=-1, # Set in Step 7 after GPU ID discovery
|
||||
node_id=bundle["node_id"],
|
||||
bundle_id_idx=bundle["bundle_id_idx"],
|
||||
)
|
||||
self.ray_worker_handles.append(handle)
|
||||
|
||||
# Step 6: Discover GPU IDs assigned to each worker via Ray runtime context.
|
||||
worker_node_and_gpu_ids = ray.get(
|
||||
[h.actor.get_node_and_gpu_ids.remote() for h in self.ray_worker_handles]
|
||||
)
|
||||
|
||||
node_workers: dict[str, list[int]] = defaultdict(list)
|
||||
node_gpus: dict[str, list[int]] = defaultdict(list)
|
||||
for i, (node_id, gpu_ids) in enumerate(worker_node_and_gpu_ids):
|
||||
node_workers[node_id].append(i)
|
||||
node_gpus[node_id].extend(gpu_ids)
|
||||
for node_id, gpu_ids in node_gpus.items():
|
||||
node_gpus[node_id] = sorted(gpu_ids)
|
||||
|
||||
# Step 7: Initialize workers with correct local_rank and
|
||||
# CUDA_VISIBLE_DEVICES. Each worker sees all GPUs assigned to
|
||||
# this executor on its node; local_rank indexes into that set.
|
||||
init_worker_refs = []
|
||||
for i, (node_id, _) in enumerate(worker_node_and_gpu_ids):
|
||||
local_rank = node_workers[node_id].index(i)
|
||||
worker_env_vars = {
|
||||
current_platform.device_control_env_var: ",".join(
|
||||
map(str, node_gpus[node_id])
|
||||
),
|
||||
}
|
||||
self.ray_worker_handles[i].local_rank = local_rank
|
||||
init_worker_refs.append(
|
||||
self.ray_worker_handles[i].actor.initialize_worker.remote(
|
||||
local_rank, worker_env_vars, self.driver_env_vars
|
||||
)
|
||||
)
|
||||
ray.get(init_worker_refs)
|
||||
|
||||
# Step 8: Collect response MQ handles
|
||||
init_results = ray.get(
|
||||
[h.actor.wait_for_init.remote() for h in self.ray_worker_handles]
|
||||
)
|
||||
|
||||
self.response_mqs: list[MessageQueue] = []
|
||||
for i, result in enumerate(init_results):
|
||||
if result["status"] != RayWorkerProc.READY_STR:
|
||||
raise RuntimeError(f"Worker {i} failed to initialize: {result}")
|
||||
self.response_mqs.append(
|
||||
MessageQueue.create_from_handle(result["handle"], 0)
|
||||
)
|
||||
|
||||
# Step 9: Start run() before wait_until_ready() to avoid
|
||||
# deadlock — workers send subscriptions inside run().
|
||||
for handle in self.ray_worker_handles:
|
||||
handle.run()
|
||||
|
||||
# Step 10: wait_until_ready() barrier
|
||||
self.rpc_broadcast_mq.wait_until_ready()
|
||||
for response_mq in self.response_mqs:
|
||||
response_mq.wait_until_ready()
|
||||
|
||||
self.futures_queue = deque[tuple[FutureWrapper, Any]]()
|
||||
self._post_init_executor()
|
||||
|
||||
self.start_worker_monitor()
|
||||
self.output_rank = self._get_output_rank()
|
||||
|
||||
def start_worker_monitor(self, inline=False) -> None:
|
||||
"""Monitor worker liveness via ray.wait() on run() ObjectRefs."""
|
||||
run_refs = [h.run_ref for h in self.ray_worker_handles if h.run_ref is not None]
|
||||
if not run_refs:
|
||||
raise RuntimeError("Ray workers have not started successfully.")
|
||||
|
||||
self_ref = weakref.ref(self)
|
||||
ref_to_rank = {
|
||||
h.run_ref: h.rank for h in self.ray_worker_handles if h.run_ref is not None
|
||||
}
|
||||
|
||||
def _should_stop() -> bool:
|
||||
executor = self_ref()
|
||||
return not executor or executor.shutting_down
|
||||
|
||||
def monitor_workers():
|
||||
# Poll with a timeout rather than blocking on ray.wait()
|
||||
# because a blocking call would segfault if Ray is torn down
|
||||
# while this thread is inside it.
|
||||
while not _should_stop() and ray.is_initialized():
|
||||
try:
|
||||
done, _ = ray.wait(run_refs, num_returns=1, timeout=5.0)
|
||||
except Exception:
|
||||
logger.exception(
|
||||
"RayWorkerMonitor: unexpected error, exiting monitor thread"
|
||||
)
|
||||
return
|
||||
if not done or _should_stop():
|
||||
continue
|
||||
|
||||
dead_ranks = [ref_to_rank[r] for r in done]
|
||||
executor = self_ref()
|
||||
if not executor:
|
||||
return
|
||||
executor.is_failed = True
|
||||
logger.error(
|
||||
"RayWorkerProc rank=%s died unexpectedly, shutting down executor.",
|
||||
dead_ranks,
|
||||
)
|
||||
executor.shutdown()
|
||||
if executor.failure_callback is not None:
|
||||
callback = executor.failure_callback
|
||||
executor.failure_callback = None
|
||||
callback()
|
||||
return
|
||||
|
||||
t = threading.Thread(
|
||||
target=monitor_workers, daemon=True, name="RayWorkerMonitor"
|
||||
)
|
||||
t.start()
|
||||
self._monitor_thread = t
|
||||
|
||||
def _join_monitor_thread(self) -> None:
|
||||
"""Wait for the monitor thread to exit.
|
||||
|
||||
Must be called before tearing down Ray resources — the monitor
|
||||
may be inside ray.wait() which would segfault if Ray is shut
|
||||
down underneath it. When the monitor itself calls shutdown()
|
||||
on worker death, we skip the join because the thread is about
|
||||
to return anyway.
|
||||
"""
|
||||
monitor = getattr(self, "_monitor_thread", None)
|
||||
if (
|
||||
monitor is not None
|
||||
and monitor.is_alive()
|
||||
and threading.current_thread() is not monitor
|
||||
):
|
||||
monitor.join(timeout=10)
|
||||
|
||||
def shutdown(self) -> None:
|
||||
"""Properly shut down the executor and its workers."""
|
||||
lock = getattr(self, "shutdown_lock", None)
|
||||
if lock is None:
|
||||
return
|
||||
|
||||
with lock:
|
||||
if getattr(self, "shutting_down", False):
|
||||
return
|
||||
self.shutting_down = True
|
||||
|
||||
self._join_monitor_thread()
|
||||
|
||||
for handle in getattr(self, "ray_worker_handles", []):
|
||||
try:
|
||||
ray.kill(handle.actor)
|
||||
logger.debug("Killed actor rank=%d", handle.rank)
|
||||
except Exception:
|
||||
logger.exception("Failed to kill actor rank=%d", handle.rank)
|
||||
|
||||
if rpc_broadcast_mq := getattr(self, "rpc_broadcast_mq", None):
|
||||
rpc_broadcast_mq.shutdown()
|
||||
self.rpc_broadcast_mq = None
|
||||
|
||||
for mq in getattr(self, "response_mqs", []):
|
||||
mq.shutdown()
|
||||
self.response_mqs = []
|
||||
+155
-14
@@ -26,6 +26,17 @@ if TYPE_CHECKING:
|
||||
logger = init_logger(__name__)
|
||||
PG_WAIT_TIMEOUT = 1800
|
||||
|
||||
# Env vars that are worker-specific and must NOT be copied from the
|
||||
# driver to Ray workers — they are set per-worker after GPU discovery.
|
||||
WORKER_SPECIFIC_ENV_VARS: set[str] = {
|
||||
"VLLM_HOST_IP",
|
||||
"VLLM_HOST_PORT",
|
||||
"LOCAL_RANK",
|
||||
"CUDA_VISIBLE_DEVICES",
|
||||
"HIP_VISIBLE_DEVICES",
|
||||
"ROCR_VISIBLE_DEVICES",
|
||||
}
|
||||
|
||||
try:
|
||||
import ray
|
||||
from ray.util import placement_group_table
|
||||
@@ -51,6 +62,8 @@ try:
|
||||
# that thread.
|
||||
self.compiled_dag_cuda_device_set = False
|
||||
|
||||
rpc_rank: int
|
||||
|
||||
def adjust_rank(self, rank_mapping: dict[int, int]) -> None:
|
||||
"""
|
||||
Adjust the rpc_rank based on the given mapping.
|
||||
@@ -214,13 +227,17 @@ def assert_ray_available():
|
||||
|
||||
|
||||
def _verify_bundles(
|
||||
placement_group: "PlacementGroup", parallel_config: ParallelConfig, device_str: str
|
||||
placement_group: "PlacementGroup",
|
||||
parallel_config: ParallelConfig,
|
||||
device_str: str,
|
||||
require_gpu_on_driver: bool = True,
|
||||
):
|
||||
"""Verify a given placement group has bundles located in the right place.
|
||||
|
||||
There are 2 rules.
|
||||
- Warn if all tensor parallel workers cannot fit in a single node.
|
||||
- Fail if driver node is not included in a placement group.
|
||||
- Fail if driver node is not included in a placement group
|
||||
(only when require_gpu_on_driver is True).
|
||||
"""
|
||||
assert ray.is_initialized(), (
|
||||
"Ray is not initialized although distributed-executor-backend is ray."
|
||||
@@ -237,7 +254,7 @@ def _verify_bundles(
|
||||
node_id_to_bundle[node_id].append(bundles[bundle_idx])
|
||||
driver_node_id = ray.get_runtime_context().get_node_id()
|
||||
|
||||
if driver_node_id not in node_id_to_bundle:
|
||||
if require_gpu_on_driver and driver_node_id not in node_id_to_bundle:
|
||||
raise RuntimeError(
|
||||
f"driver node id {driver_node_id} is not included in a placement "
|
||||
f"group {placement_group.id}. Node id -> bundles "
|
||||
@@ -266,6 +283,115 @@ def _verify_bundles(
|
||||
)
|
||||
|
||||
|
||||
def build_actor_name(
|
||||
instance_id: str,
|
||||
rank: int,
|
||||
tp_size: int,
|
||||
pp_size: int,
|
||||
pcp_size: int,
|
||||
) -> str:
|
||||
"""Build a descriptive Ray actor name for dashboard visibility."""
|
||||
name = f"vllm_Worker_{instance_id}"
|
||||
if tp_size > 1:
|
||||
name += f"_TP{rank % tp_size}"
|
||||
if pp_size > 1:
|
||||
name += f"_PP{(rank // tp_size) % pp_size}"
|
||||
if pcp_size > 1:
|
||||
name += f"_PCP{rank // (tp_size * pp_size)}"
|
||||
return name
|
||||
|
||||
|
||||
def get_bundles_for_indices(
|
||||
placement_group: "PlacementGroup",
|
||||
bundle_indices: list[int],
|
||||
world_size: int,
|
||||
) -> list[tuple[int, str, str]]:
|
||||
"""
|
||||
Return GPU bundle indices paired with node IDs and node IPs for
|
||||
explicit bundle indices specified via VLLM_RAY_BUNDLE_INDICES.
|
||||
"""
|
||||
assert len(bundle_indices) == world_size, (
|
||||
"VLLM_RAY_BUNDLE_INDICES must have the same size"
|
||||
f" as the world size, but got {bundle_indices=} "
|
||||
f"and {world_size=}"
|
||||
)
|
||||
assert len(set(bundle_indices)) == len(bundle_indices), (
|
||||
"VLLM_RAY_BUNDLE_INDICES cannot have duplicate values,"
|
||||
f" but got {bundle_indices=}"
|
||||
)
|
||||
|
||||
pg_data = placement_group_table(placement_group)
|
||||
pg_bundle_to_node = pg_data["bundles_to_node_id"]
|
||||
node_id_to_ip = {
|
||||
n["NodeID"]: n["NodeManagerAddress"] for n in ray.nodes() if n["Alive"]
|
||||
}
|
||||
return [
|
||||
(bid, pg_bundle_to_node[bid], node_id_to_ip[pg_bundle_to_node[bid]])
|
||||
for bid in bundle_indices
|
||||
]
|
||||
|
||||
|
||||
def get_bundles_sorted_by_node(
|
||||
placement_group: "PlacementGroup",
|
||||
) -> list[tuple[int, str, str]]:
|
||||
"""
|
||||
Return GPU bundle indices paired with node IDs and node IPs,
|
||||
sorted driver-first.
|
||||
|
||||
This utility has to be invoked from the driver node.
|
||||
|
||||
Example: 3-node cluster, driver on node-A, PG bundles spread
|
||||
across nodes:
|
||||
|
||||
Input: [
|
||||
(0, node-C),
|
||||
(1, node-A),
|
||||
(2, node-B),
|
||||
(3, node-C),
|
||||
(4, node-A),
|
||||
(5, node-B),
|
||||
]
|
||||
Output: [
|
||||
(1, node-A),
|
||||
(4, node-A),
|
||||
(2, node-B),
|
||||
(5, node-B),
|
||||
(0, node-C),
|
||||
(3, node-C),
|
||||
]
|
||||
"""
|
||||
pg_data = placement_group_table(placement_group)
|
||||
bundle_to_node = pg_data["bundles_to_node_id"]
|
||||
|
||||
ray_device_key = current_platform.ray_device_key
|
||||
if not ray_device_key:
|
||||
raise ValueError(
|
||||
f"current platform {current_platform.device_name} does not support ray."
|
||||
)
|
||||
|
||||
node_id_to_ip = {
|
||||
n["NodeID"]: n["NodeManagerAddress"] for n in ray.nodes() if n["Alive"]
|
||||
}
|
||||
|
||||
bundle_specs = placement_group.bundle_specs
|
||||
assert bundle_specs is not None
|
||||
bundle_to_node_id: list[tuple[int, str, str]] = []
|
||||
for bundle_idx, bundle in enumerate(bundle_specs):
|
||||
if bundle.get(ray_device_key):
|
||||
node_id = bundle_to_node.get(bundle_idx)
|
||||
bundle_to_node_id.append((bundle_idx, node_id, node_id_to_ip[node_id]))
|
||||
|
||||
driver_node = ray.get_runtime_context().get_node_id()
|
||||
|
||||
def _sort_key(item):
|
||||
_, node_id, _ = item
|
||||
return (0 if node_id == driver_node else 1, node_id)
|
||||
|
||||
bundle_to_node_id.sort(key=_sort_key)
|
||||
|
||||
return bundle_to_node_id
|
||||
|
||||
|
||||
def _wait_until_pg_ready(current_placement_group: "PlacementGroup"):
|
||||
"""Wait until a placement group is ready.
|
||||
|
||||
@@ -352,6 +478,7 @@ def _wait_until_pg_removed(current_placement_group: "PlacementGroup"):
|
||||
def initialize_ray_cluster(
|
||||
parallel_config: ParallelConfig,
|
||||
ray_address: str | None = None,
|
||||
require_gpu_on_driver: bool = True,
|
||||
):
|
||||
"""Initialize the distributed cluster with Ray.
|
||||
|
||||
@@ -363,10 +490,18 @@ def initialize_ray_cluster(
|
||||
parallel_config: The configurations for parallel execution.
|
||||
ray_address: The address of the Ray cluster. If None, uses
|
||||
the default Ray cluster address.
|
||||
require_gpu_on_driver: If True (default), require at least one GPU
|
||||
on the current (driver) node and pin the first PG bundle to it.
|
||||
Set to False for executors like RayExecutorV2 where all GPU work
|
||||
is delegated to remote Ray actors.
|
||||
"""
|
||||
assert_ray_available()
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
# Disable Ray usage stats collection
|
||||
if os.environ.get("RAY_USAGE_STATS_ENABLED", "0") != "1":
|
||||
os.environ["RAY_USAGE_STATS_ENABLED"] = "0"
|
||||
|
||||
# Prevalidate GPU requirements before Ray processing
|
||||
if current_platform.is_cuda() and parallel_config.world_size > 1:
|
||||
available_gpus = current_platform.device_count()
|
||||
@@ -459,16 +594,20 @@ def initialize_ray_cluster(
|
||||
current_ip = get_ip()
|
||||
current_node_id = ray.get_runtime_context().get_node_id()
|
||||
current_node_resource = available_resources_per_node()[current_node_id]
|
||||
if current_node_resource.get(device_str, 0) < 1:
|
||||
raise ValueError(
|
||||
f"Current node has no {device_str} available. "
|
||||
f"{current_node_resource=}. vLLM engine cannot start without "
|
||||
f"{device_str}. Make sure you have at least 1 {device_str} "
|
||||
f"available in a node {current_node_id=} {current_ip=}."
|
||||
)
|
||||
# This way, at least bundle is required to be created in a current
|
||||
# node.
|
||||
placement_group_specs[0][f"node:{current_ip}"] = 0.001
|
||||
# TODO (jeffreywang): require_gpu_on_driver should be always False
|
||||
# after deprecating RayDistributedExecutor.
|
||||
if require_gpu_on_driver:
|
||||
if current_node_resource.get(device_str, 0) < 1:
|
||||
raise ValueError(
|
||||
f"Current node has no {device_str} available. "
|
||||
f"{current_node_resource=}. vLLM engine cannot start "
|
||||
f"without {device_str}. Make sure you have at least 1 "
|
||||
f"{device_str} available in a node "
|
||||
f"{current_node_id=} {current_ip=}."
|
||||
)
|
||||
# This way, at least bundle is required to be created in a
|
||||
# current node.
|
||||
placement_group_specs[0][f"node:{current_ip}"] = 0.001
|
||||
|
||||
# By default, Ray packs resources as much as possible.
|
||||
current_placement_group = ray.util.placement_group(
|
||||
@@ -477,7 +616,9 @@ def initialize_ray_cluster(
|
||||
_wait_until_pg_ready(current_placement_group)
|
||||
|
||||
assert current_placement_group is not None
|
||||
_verify_bundles(current_placement_group, parallel_config, device_str)
|
||||
_verify_bundles(
|
||||
current_placement_group, parallel_config, device_str, require_gpu_on_driver
|
||||
)
|
||||
# Set the placement group in the parallel config
|
||||
parallel_config.placement_group = current_placement_group
|
||||
|
||||
|
||||
Reference in New Issue
Block a user