[Core] Remove the legacy Ray compiled-graph executor (#61712)

Signed-off-by: Woosuk Kwon <woosuk@inferact.ai>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Woosuk Kwon
2026-10-14 18:13:27 -07:00
committed by GitHub
co-authored by Claude Opus 5.5
parent bfdbfa1632
commit e57bb4ace7
16 changed files with 14 additions and 1257 deletions
-219
View File
@@ -4,26 +4,11 @@
import os
import time
from collections import defaultdict
from concurrent.futures import Future
from typing import TYPE_CHECKING, Union
import numpy as np
import vllm.platforms
from vllm.config import ParallelConfig
from vllm.distributed import get_pp_group
from vllm.distributed.kv_transfer.kv_connector.utils import KVOutputAggregator
from vllm.logger import init_logger
from vllm.platforms import current_platform
from vllm.sequence import IntermediateTensors
from vllm.utils.network_utils import get_ip
from vllm.v1.outputs import AsyncModelRunnerOutput
from vllm.v1.serial_utils import run_method
from vllm.v1.worker.worker_base import WorkerWrapperBase
if TYPE_CHECKING:
from vllm.v1.core.sched.output import GrammarOutput, SchedulerOutput
from vllm.v1.outputs import ModelRunnerOutput
logger = init_logger(__name__)
PG_WAIT_TIMEOUT = 1800
@@ -53,134 +38,6 @@ try:
available_resources_per_node = _state._available_resources_per_node
class RayWorkerWrapper(WorkerWrapperBase):
"""Ray wrapper for vllm.worker.Worker, allowing Worker to be
lazily initialized after Ray sets CUDA_VISIBLE_DEVICES."""
def __init__(self, *args, **kwargs) -> None:
super().__init__(*args, **kwargs)
# Since the compiled DAG runs a main execution
# in a different thread that calls cuda.set_device.
# The flag indicates is set_device is called on
# that thread.
self.compiled_dag_cuda_device_set = False
rpc_rank: int
def adjust_rank(self, rank_mapping: dict[int, int]) -> None:
"""Adjust the rpc_rank based on the given mapping.
It is only used during the initialization of the executor,
to adjust the rpc_rank of workers after we create all workers.
"""
if self.rpc_rank in rank_mapping:
self.rpc_rank = rank_mapping[self.rpc_rank]
def execute_method(self, method: str | bytes, *args, **kwargs):
try:
return run_method(self, method, args, kwargs)
except Exception as e:
# if the driver worker also execute methods,
# exceptions in the rest worker may cause deadlock in rpc
# see https://github.com/vllm-project/vllm/issues/3455
msg = (
f"Error executing method {method!r}. "
"This might cause deadlock in distributed execution."
)
logger.exception(msg)
raise e
def get_node_ip(self) -> str:
return get_ip()
def get_node_and_physical_gpu_ids(self) -> tuple[str, list[int]]:
node_id = ray.get_runtime_context().get_node_id()
device_key = vllm.platforms.current_platform.ray_device_key
if not device_key:
raise RuntimeError(
"current platform %s does not support ray.",
vllm.platforms.current_platform.device_name,
)
physical_gpu_ids = ray.get_runtime_context().get_accelerator_ids()[
device_key
]
return node_id, physical_gpu_ids
def setup_device_if_necessary(self):
# TODO(swang): This is needed right now because Ray CG executes
# on a background thread, so we need to reset torch's current
# device.
# We can remove this API after it is fixed in compiled graph.
assert self.worker is not None, "Worker is not initialized"
if not self.compiled_dag_cuda_device_set:
if current_platform.is_tpu():
# Not needed
pass
else:
assert self.worker.device is not None
current_platform.set_device(self.worker.device)
self.compiled_dag_cuda_device_set = True
def execute_model_ray(
self,
execute_model_input: tuple["SchedulerOutput", "GrammarOutput"]
| tuple["SchedulerOutput", "GrammarOutput", "IntermediateTensors"],
) -> Union[
"ModelRunnerOutput",
tuple["SchedulerOutput", "GrammarOutput", "IntermediateTensors"],
]:
# This method is used by Ray Compiled Graph to execute the model,
# and it needs a special logic of self.setup_device_if_necessary()
self.setup_device_if_necessary()
assert self.worker is not None, "Worker is not initialized"
if len(execute_model_input) == 3:
scheduler_output, grammar_output, intermediate_tensors = (
execute_model_input
)
else:
scheduler_output, grammar_output = execute_model_input
intermediate_tensors = None
assert self.worker.model_runner is not None
output = self.worker.model_runner.execute_model(
scheduler_output, intermediate_tensors
)
if self._is_intermediate_tensors(output):
if (
self.worker.model_runner.supports_mm_inputs
and get_pp_group().is_first_rank
):
# Strip mm_features before Ray forwards it to the next PP Stage.
# PP Stage>0 only needs the intermediate tensors,
# not preprocessed multimodal data.
# scheduled_new_reqs is a required field of SchedulerOutput,
# so accessing it directly will raise AttributeError if missing.
for req in scheduler_output.scheduled_new_reqs:
req.mm_features = []
return scheduler_output, grammar_output, output
if isinstance(output, AsyncModelRunnerOutput):
output = output.get_output()
if not self._is_last_rank():
# Case where there are no scheduled requests
# but may still be finished requests.
assert not output or not output.req_ids
output = scheduler_output, grammar_output, None
elif output is None:
output = self.worker.model_runner.sample_tokens(grammar_output)
# Ensure outputs crossing Ray compiled DAG are serializable.
# AsyncModelRunnerOutput holds CUDA events and cannot be
# pickled.
if isinstance(output, AsyncModelRunnerOutput):
output = output.get_output()
return output
def _is_intermediate_tensors(self, output) -> bool:
return isinstance(output, IntermediateTensors)
def _is_last_rank(self) -> bool:
return get_pp_group().is_last_rank
ray_import_err = None
except ImportError as e:
@@ -188,80 +45,6 @@ except ImportError as e:
# only capture string to avoid variable references in the traceback that can
# prevent garbage collection in some cases
ray_import_err = str(e)
RayWorkerWrapper = None # type: ignore
def detach_zero_copy_from_model_runner_output(output: "ModelRunnerOutput") -> None:
"""Detach Ray SHM-channel zero-copy buffers from a ModelRunnerOutput in-place.
Ray compiled DAG SHM channels may return zero-copy objects (e.g. `np.ndarray`)
backed by Ray's shared-memory object store. Ray's channel docs explicitly
warn that subsequent reads may block if such an object is still in scope.
vLLM can return numpy-backed logprobs and routed experts in
`ModelRunnerOutput`. If those arrays are backed by Ray SHM (commonly
read-only), retaining them in scope across scheduler iterations can stall
the channel and eventually hit `RAY_CGRAPH_get_timeout`.
Copy read-only numpy arrays so the returned output no longer retains
references to Ray's shared-memory buffers.
We intentionally do not touch `prompt_logprobs_dict`: those entries are
`LogprobsTensors` backed by PyTorch-owned CPU tensors (`to_cpu_nonblocking`
or `empty_cpu`), not NumPy views decoded from Ray channels.
"""
def _copy_if_readonly(arr):
if isinstance(arr, np.ndarray) and not arr.flags.writeable:
return arr.copy()
return arr
if output.logprobs is not None:
token_ids, logprobs, ranks, cu_num_generated_tokens = output.logprobs
# `cu_num_generated_tokens` is already a plain Python list (or None),
# so it never aliases Ray SHM buffers and can be reused as-is.
token_ids_c = _copy_if_readonly(token_ids)
logprobs_c = _copy_if_readonly(logprobs)
ranks_c = _copy_if_readonly(ranks)
if (
token_ids_c is not token_ids
or logprobs_c is not logprobs
or ranks_c is not ranks
):
output.logprobs = type(output.logprobs)(
token_ids_c, logprobs_c, ranks_c, cu_num_generated_tokens
)
aux_output = output.aux_output_connector_output
if aux_output is not None:
for request_output in aux_output.values():
request_output.rows = _copy_if_readonly(request_output.rows)
class FutureWrapper(Future):
"""A wrapper around Ray output reference to meet the interface
of .execute_model(): The top level (core busy loop) expects .result() api
to block and return a single output.
If aggregator is provided, the outputs from all workers are aggregated upon
the result() call. If not only the first worker's output is returned.
"""
def __init__(self, ref_or_refs, aggregator: KVOutputAggregator | None = None):
super().__init__()
self.ref_or_refs = ref_or_refs
self.aggregator = aggregator
def result(self, timeout=None):
outputs = ray.get(self.ref_or_refs, timeout=timeout)
if self.aggregator is None:
detach_zero_copy_from_model_runner_output(outputs)
return outputs
for output in outputs:
detach_zero_copy_from_model_runner_output(output)
return self.aggregator.aggregate(outputs, output_rank=0)
def ray_is_available() -> bool:
@@ -627,8 +410,6 @@ def initialize_ray_cluster(
current_ip = get_ip()
current_node_id = ray.get_runtime_context().get_node_id()
current_node_resource = available_resources_per_node()[current_node_id]
# TODO (jeffreywang): require_gpu_on_driver should be always False
# after deprecating RayDistributedExecutor.
if require_gpu_on_driver:
if current_node_resource.get(device_str, 0) < 1:
raise ValueError(