[Core] Remove the legacy Ray compiled-graph executor (#61712)
Signed-off-by: Woosuk Kwon <woosuk@inferact.ai> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
bfdbfa1632
commit
e57bb4ace7
@@ -4,26 +4,11 @@
|
||||
import os
|
||||
import time
|
||||
from collections import defaultdict
|
||||
from concurrent.futures import Future
|
||||
from typing import TYPE_CHECKING, Union
|
||||
|
||||
import numpy as np
|
||||
|
||||
import vllm.platforms
|
||||
from vllm.config import ParallelConfig
|
||||
from vllm.distributed import get_pp_group
|
||||
from vllm.distributed.kv_transfer.kv_connector.utils import KVOutputAggregator
|
||||
from vllm.logger import init_logger
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.utils.network_utils import get_ip
|
||||
from vllm.v1.outputs import AsyncModelRunnerOutput
|
||||
from vllm.v1.serial_utils import run_method
|
||||
from vllm.v1.worker.worker_base import WorkerWrapperBase
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.v1.core.sched.output import GrammarOutput, SchedulerOutput
|
||||
from vllm.v1.outputs import ModelRunnerOutput
|
||||
|
||||
logger = init_logger(__name__)
|
||||
PG_WAIT_TIMEOUT = 1800
|
||||
@@ -53,134 +38,6 @@ try:
|
||||
|
||||
available_resources_per_node = _state._available_resources_per_node
|
||||
|
||||
class RayWorkerWrapper(WorkerWrapperBase):
|
||||
"""Ray wrapper for vllm.worker.Worker, allowing Worker to be
|
||||
lazily initialized after Ray sets CUDA_VISIBLE_DEVICES."""
|
||||
|
||||
def __init__(self, *args, **kwargs) -> None:
|
||||
super().__init__(*args, **kwargs)
|
||||
# Since the compiled DAG runs a main execution
|
||||
# in a different thread that calls cuda.set_device.
|
||||
# The flag indicates is set_device is called on
|
||||
# that thread.
|
||||
self.compiled_dag_cuda_device_set = False
|
||||
|
||||
rpc_rank: int
|
||||
|
||||
def adjust_rank(self, rank_mapping: dict[int, int]) -> None:
|
||||
"""Adjust the rpc_rank based on the given mapping.
|
||||
It is only used during the initialization of the executor,
|
||||
to adjust the rpc_rank of workers after we create all workers.
|
||||
"""
|
||||
if self.rpc_rank in rank_mapping:
|
||||
self.rpc_rank = rank_mapping[self.rpc_rank]
|
||||
|
||||
def execute_method(self, method: str | bytes, *args, **kwargs):
|
||||
try:
|
||||
return run_method(self, method, args, kwargs)
|
||||
except Exception as e:
|
||||
# if the driver worker also execute methods,
|
||||
# exceptions in the rest worker may cause deadlock in rpc
|
||||
# see https://github.com/vllm-project/vllm/issues/3455
|
||||
msg = (
|
||||
f"Error executing method {method!r}. "
|
||||
"This might cause deadlock in distributed execution."
|
||||
)
|
||||
logger.exception(msg)
|
||||
raise e
|
||||
|
||||
def get_node_ip(self) -> str:
|
||||
return get_ip()
|
||||
|
||||
def get_node_and_physical_gpu_ids(self) -> tuple[str, list[int]]:
|
||||
node_id = ray.get_runtime_context().get_node_id()
|
||||
device_key = vllm.platforms.current_platform.ray_device_key
|
||||
if not device_key:
|
||||
raise RuntimeError(
|
||||
"current platform %s does not support ray.",
|
||||
vllm.platforms.current_platform.device_name,
|
||||
)
|
||||
physical_gpu_ids = ray.get_runtime_context().get_accelerator_ids()[
|
||||
device_key
|
||||
]
|
||||
return node_id, physical_gpu_ids
|
||||
|
||||
def setup_device_if_necessary(self):
|
||||
# TODO(swang): This is needed right now because Ray CG executes
|
||||
# on a background thread, so we need to reset torch's current
|
||||
# device.
|
||||
# We can remove this API after it is fixed in compiled graph.
|
||||
assert self.worker is not None, "Worker is not initialized"
|
||||
if not self.compiled_dag_cuda_device_set:
|
||||
if current_platform.is_tpu():
|
||||
# Not needed
|
||||
pass
|
||||
else:
|
||||
assert self.worker.device is not None
|
||||
current_platform.set_device(self.worker.device)
|
||||
|
||||
self.compiled_dag_cuda_device_set = True
|
||||
|
||||
def execute_model_ray(
|
||||
self,
|
||||
execute_model_input: tuple["SchedulerOutput", "GrammarOutput"]
|
||||
| tuple["SchedulerOutput", "GrammarOutput", "IntermediateTensors"],
|
||||
) -> Union[
|
||||
"ModelRunnerOutput",
|
||||
tuple["SchedulerOutput", "GrammarOutput", "IntermediateTensors"],
|
||||
]:
|
||||
# This method is used by Ray Compiled Graph to execute the model,
|
||||
# and it needs a special logic of self.setup_device_if_necessary()
|
||||
self.setup_device_if_necessary()
|
||||
assert self.worker is not None, "Worker is not initialized"
|
||||
if len(execute_model_input) == 3:
|
||||
scheduler_output, grammar_output, intermediate_tensors = (
|
||||
execute_model_input
|
||||
)
|
||||
else:
|
||||
scheduler_output, grammar_output = execute_model_input
|
||||
intermediate_tensors = None
|
||||
assert self.worker.model_runner is not None
|
||||
output = self.worker.model_runner.execute_model(
|
||||
scheduler_output, intermediate_tensors
|
||||
)
|
||||
if self._is_intermediate_tensors(output):
|
||||
if (
|
||||
self.worker.model_runner.supports_mm_inputs
|
||||
and get_pp_group().is_first_rank
|
||||
):
|
||||
# Strip mm_features before Ray forwards it to the next PP Stage.
|
||||
# PP Stage>0 only needs the intermediate tensors,
|
||||
# not preprocessed multimodal data.
|
||||
|
||||
# scheduled_new_reqs is a required field of SchedulerOutput,
|
||||
# so accessing it directly will raise AttributeError if missing.
|
||||
for req in scheduler_output.scheduled_new_reqs:
|
||||
req.mm_features = []
|
||||
return scheduler_output, grammar_output, output
|
||||
|
||||
if isinstance(output, AsyncModelRunnerOutput):
|
||||
output = output.get_output()
|
||||
if not self._is_last_rank():
|
||||
# Case where there are no scheduled requests
|
||||
# but may still be finished requests.
|
||||
assert not output or not output.req_ids
|
||||
output = scheduler_output, grammar_output, None
|
||||
elif output is None:
|
||||
output = self.worker.model_runner.sample_tokens(grammar_output)
|
||||
# Ensure outputs crossing Ray compiled DAG are serializable.
|
||||
# AsyncModelRunnerOutput holds CUDA events and cannot be
|
||||
# pickled.
|
||||
if isinstance(output, AsyncModelRunnerOutput):
|
||||
output = output.get_output()
|
||||
return output
|
||||
|
||||
def _is_intermediate_tensors(self, output) -> bool:
|
||||
return isinstance(output, IntermediateTensors)
|
||||
|
||||
def _is_last_rank(self) -> bool:
|
||||
return get_pp_group().is_last_rank
|
||||
|
||||
ray_import_err = None
|
||||
|
||||
except ImportError as e:
|
||||
@@ -188,80 +45,6 @@ except ImportError as e:
|
||||
# only capture string to avoid variable references in the traceback that can
|
||||
# prevent garbage collection in some cases
|
||||
ray_import_err = str(e)
|
||||
RayWorkerWrapper = None # type: ignore
|
||||
|
||||
|
||||
def detach_zero_copy_from_model_runner_output(output: "ModelRunnerOutput") -> None:
|
||||
"""Detach Ray SHM-channel zero-copy buffers from a ModelRunnerOutput in-place.
|
||||
|
||||
Ray compiled DAG SHM channels may return zero-copy objects (e.g. `np.ndarray`)
|
||||
backed by Ray's shared-memory object store. Ray's channel docs explicitly
|
||||
warn that subsequent reads may block if such an object is still in scope.
|
||||
|
||||
vLLM can return numpy-backed logprobs and routed experts in
|
||||
`ModelRunnerOutput`. If those arrays are backed by Ray SHM (commonly
|
||||
read-only), retaining them in scope across scheduler iterations can stall
|
||||
the channel and eventually hit `RAY_CGRAPH_get_timeout`.
|
||||
|
||||
Copy read-only numpy arrays so the returned output no longer retains
|
||||
references to Ray's shared-memory buffers.
|
||||
|
||||
We intentionally do not touch `prompt_logprobs_dict`: those entries are
|
||||
`LogprobsTensors` backed by PyTorch-owned CPU tensors (`to_cpu_nonblocking`
|
||||
or `empty_cpu`), not NumPy views decoded from Ray channels.
|
||||
"""
|
||||
|
||||
def _copy_if_readonly(arr):
|
||||
if isinstance(arr, np.ndarray) and not arr.flags.writeable:
|
||||
return arr.copy()
|
||||
return arr
|
||||
|
||||
if output.logprobs is not None:
|
||||
token_ids, logprobs, ranks, cu_num_generated_tokens = output.logprobs
|
||||
|
||||
# `cu_num_generated_tokens` is already a plain Python list (or None),
|
||||
# so it never aliases Ray SHM buffers and can be reused as-is.
|
||||
token_ids_c = _copy_if_readonly(token_ids)
|
||||
logprobs_c = _copy_if_readonly(logprobs)
|
||||
ranks_c = _copy_if_readonly(ranks)
|
||||
if (
|
||||
token_ids_c is not token_ids
|
||||
or logprobs_c is not logprobs
|
||||
or ranks_c is not ranks
|
||||
):
|
||||
output.logprobs = type(output.logprobs)(
|
||||
token_ids_c, logprobs_c, ranks_c, cu_num_generated_tokens
|
||||
)
|
||||
|
||||
aux_output = output.aux_output_connector_output
|
||||
if aux_output is not None:
|
||||
for request_output in aux_output.values():
|
||||
request_output.rows = _copy_if_readonly(request_output.rows)
|
||||
|
||||
|
||||
class FutureWrapper(Future):
|
||||
"""A wrapper around Ray output reference to meet the interface
|
||||
of .execute_model(): The top level (core busy loop) expects .result() api
|
||||
to block and return a single output.
|
||||
|
||||
If aggregator is provided, the outputs from all workers are aggregated upon
|
||||
the result() call. If not only the first worker's output is returned.
|
||||
"""
|
||||
|
||||
def __init__(self, ref_or_refs, aggregator: KVOutputAggregator | None = None):
|
||||
super().__init__()
|
||||
self.ref_or_refs = ref_or_refs
|
||||
self.aggregator = aggregator
|
||||
|
||||
def result(self, timeout=None):
|
||||
outputs = ray.get(self.ref_or_refs, timeout=timeout)
|
||||
if self.aggregator is None:
|
||||
detach_zero_copy_from_model_runner_output(outputs)
|
||||
return outputs
|
||||
|
||||
for output in outputs:
|
||||
detach_zero_copy_from_model_runner_output(output)
|
||||
return self.aggregator.aggregate(outputs, output_rank=0)
|
||||
|
||||
|
||||
def ray_is_available() -> bool:
|
||||
@@ -627,8 +410,6 @@ def initialize_ray_cluster(
|
||||
current_ip = get_ip()
|
||||
current_node_id = ray.get_runtime_context().get_node_id()
|
||||
current_node_resource = available_resources_per_node()[current_node_id]
|
||||
# TODO (jeffreywang): require_gpu_on_driver should be always False
|
||||
# after deprecating RayDistributedExecutor.
|
||||
if require_gpu_on_driver:
|
||||
if current_node_resource.get(device_str, 0) < 1:
|
||||
raise ValueError(
|
||||
|
||||
Reference in New Issue
Block a user