[V1] Support DP with Ray (#18779)

This commit is contained in:
Rui Qiao
2025-06-02 21:15:13 -07:00
committed by GitHub
parent 9e6f61e8c3
commit bdce64f236
10 changed files with 551 additions and 120 deletions
+65 -9
View File
@@ -29,9 +29,9 @@ from vllm.v1.engine.core import EngineCore, EngineCoreProc
from vllm.v1.engine.exceptions import EngineDeadError
from vllm.v1.executor.abstract import Executor
from vllm.v1.serial_utils import MsgpackDecoder, MsgpackEncoder, bytestr
from vllm.v1.utils import (CoreEngine, CoreEngineProcManager,
EngineZmqAddresses, get_engine_client_zmq_addr,
wait_for_engine_startup)
from vllm.v1.utils import (CoreEngine, CoreEngineActorManager,
CoreEngineProcManager, EngineZmqAddresses,
get_engine_client_zmq_addr, wait_for_engine_startup)
logger = init_logger(__name__)
@@ -68,6 +68,8 @@ class EngineCoreClient(ABC):
if multiprocess_mode and asyncio_mode:
if vllm_config.parallel_config.data_parallel_size > 1:
if vllm_config.parallel_config.data_parallel_backend == "ray":
return RayDPClient(vllm_config, executor_class, log_stats)
return DPAsyncMPClient(vllm_config, executor_class, log_stats)
return AsyncMPClient(vllm_config, executor_class, log_stats)
@@ -273,7 +275,10 @@ class BackgroundResources:
circular reference back to the client object."""
ctx: Union[zmq.Context]
local_engine_manager: Optional[CoreEngineProcManager] = None
# If CoreEngineProcManager, it manages local engines;
# if CoreEngineActorManager, it manages all engines.
engine_manager: Optional[Union[CoreEngineProcManager,
CoreEngineActorManager]] = None
coordinator: Optional[DPCoordinator] = None
output_socket: Optional[Union[zmq.Socket, zmq.asyncio.Socket]] = None
input_socket: Optional[Union[zmq.Socket, zmq.asyncio.Socket]] = None
@@ -290,8 +295,8 @@ class BackgroundResources:
"""Clean up background resources."""
self.engine_dead = True
if self.local_engine_manager is not None:
self.local_engine_manager.close()
if self.engine_manager is not None:
self.engine_manager.close()
if self.coordinator is not None:
self.coordinator.close()
@@ -457,7 +462,7 @@ class MPClient(EngineCoreClient):
if local_engine_count:
# In server mode, start_index and local_start_index will
# both be 0.
self.resources.local_engine_manager = CoreEngineProcManager(
self.resources.engine_manager = CoreEngineProcManager(
EngineCoreProc.run_engine_core,
vllm_config=vllm_config,
executor_class=executor_class,
@@ -484,13 +489,18 @@ class MPClient(EngineCoreClient):
addresses.coordinator_input, addresses.coordinator_output = (
coordinator.get_engine_socket_addresses())
proc_manager = self.resources.engine_manager
assert isinstance(proc_manager, (type(None), CoreEngineProcManager)), (
"_wait_for_engine_startup should only be "
"called with CoreEngineProcManager")
wait_for_engine_startup(
handshake_socket,
addresses,
self.core_engines,
self.vllm_config.parallel_config,
self.vllm_config.cache_config,
self.resources.local_engine_manager,
proc_manager,
coordinator.proc if coordinator else None,
)
@@ -887,7 +897,6 @@ class DPAsyncMPClient(AsyncMPClient):
log_stats: bool,
client_addresses: Optional[dict[str, str]] = None,
client_index: int = 0):
self.current_wave = 0
self.engines_running = False
# To route aborts to the correct engine.
@@ -1050,3 +1059,50 @@ class DPAsyncMPClient(AsyncMPClient):
if not self.resources.engine_dead:
await self._send_input(EngineCoreRequestType.ABORT, request_ids,
engine)
class RayDPClient(DPAsyncMPClient):
"""
Ray-based client for multi-proc, multi-engine (data parallel)
EngineCore.
"""
def __init__(
self,
vllm_config: VllmConfig,
executor_class: type[Executor],
log_stats: bool,
client_addresses: Optional[dict[str, str]] = None,
client_index: int = 0,
):
super().__init__(vllm_config, executor_class, log_stats,
client_addresses, client_index)
def _init_engines_direct(self, vllm_config: VllmConfig, local_only: bool,
local_start_index: int, input_address: str,
output_address: str,
executor_class: type[Executor], log_stats: bool):
"""Self-contained client mode, launch engine and coordinator process
as needed."""
parallel_config = vllm_config.parallel_config
assert parallel_config.data_parallel_rank == 0
assert local_start_index == 0
addresses = EngineZmqAddresses(
inputs=[input_address],
outputs=[output_address],
)
if len(self.core_engines) > 1:
coordinator = DPCoordinator(parallel_config)
self.resources.coordinator = coordinator
addresses.coordinator_input, addresses.coordinator_output = (
coordinator.get_engine_socket_addresses())
# Start all engines.
self.resources.engine_manager = CoreEngineActorManager(
vllm_config=vllm_config,
addresses=addresses,
executor_class=executor_class,
log_stats=log_stats)