[V1] Support DP with Ray (#18779)
This commit is contained in:
@@ -29,9 +29,9 @@ from vllm.v1.engine.core import EngineCore, EngineCoreProc
|
||||
from vllm.v1.engine.exceptions import EngineDeadError
|
||||
from vllm.v1.executor.abstract import Executor
|
||||
from vllm.v1.serial_utils import MsgpackDecoder, MsgpackEncoder, bytestr
|
||||
from vllm.v1.utils import (CoreEngine, CoreEngineProcManager,
|
||||
EngineZmqAddresses, get_engine_client_zmq_addr,
|
||||
wait_for_engine_startup)
|
||||
from vllm.v1.utils import (CoreEngine, CoreEngineActorManager,
|
||||
CoreEngineProcManager, EngineZmqAddresses,
|
||||
get_engine_client_zmq_addr, wait_for_engine_startup)
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -68,6 +68,8 @@ class EngineCoreClient(ABC):
|
||||
|
||||
if multiprocess_mode and asyncio_mode:
|
||||
if vllm_config.parallel_config.data_parallel_size > 1:
|
||||
if vllm_config.parallel_config.data_parallel_backend == "ray":
|
||||
return RayDPClient(vllm_config, executor_class, log_stats)
|
||||
return DPAsyncMPClient(vllm_config, executor_class, log_stats)
|
||||
|
||||
return AsyncMPClient(vllm_config, executor_class, log_stats)
|
||||
@@ -273,7 +275,10 @@ class BackgroundResources:
|
||||
circular reference back to the client object."""
|
||||
|
||||
ctx: Union[zmq.Context]
|
||||
local_engine_manager: Optional[CoreEngineProcManager] = None
|
||||
# If CoreEngineProcManager, it manages local engines;
|
||||
# if CoreEngineActorManager, it manages all engines.
|
||||
engine_manager: Optional[Union[CoreEngineProcManager,
|
||||
CoreEngineActorManager]] = None
|
||||
coordinator: Optional[DPCoordinator] = None
|
||||
output_socket: Optional[Union[zmq.Socket, zmq.asyncio.Socket]] = None
|
||||
input_socket: Optional[Union[zmq.Socket, zmq.asyncio.Socket]] = None
|
||||
@@ -290,8 +295,8 @@ class BackgroundResources:
|
||||
"""Clean up background resources."""
|
||||
|
||||
self.engine_dead = True
|
||||
if self.local_engine_manager is not None:
|
||||
self.local_engine_manager.close()
|
||||
if self.engine_manager is not None:
|
||||
self.engine_manager.close()
|
||||
if self.coordinator is not None:
|
||||
self.coordinator.close()
|
||||
|
||||
@@ -457,7 +462,7 @@ class MPClient(EngineCoreClient):
|
||||
if local_engine_count:
|
||||
# In server mode, start_index and local_start_index will
|
||||
# both be 0.
|
||||
self.resources.local_engine_manager = CoreEngineProcManager(
|
||||
self.resources.engine_manager = CoreEngineProcManager(
|
||||
EngineCoreProc.run_engine_core,
|
||||
vllm_config=vllm_config,
|
||||
executor_class=executor_class,
|
||||
@@ -484,13 +489,18 @@ class MPClient(EngineCoreClient):
|
||||
addresses.coordinator_input, addresses.coordinator_output = (
|
||||
coordinator.get_engine_socket_addresses())
|
||||
|
||||
proc_manager = self.resources.engine_manager
|
||||
assert isinstance(proc_manager, (type(None), CoreEngineProcManager)), (
|
||||
"_wait_for_engine_startup should only be "
|
||||
"called with CoreEngineProcManager")
|
||||
|
||||
wait_for_engine_startup(
|
||||
handshake_socket,
|
||||
addresses,
|
||||
self.core_engines,
|
||||
self.vllm_config.parallel_config,
|
||||
self.vllm_config.cache_config,
|
||||
self.resources.local_engine_manager,
|
||||
proc_manager,
|
||||
coordinator.proc if coordinator else None,
|
||||
)
|
||||
|
||||
@@ -887,7 +897,6 @@ class DPAsyncMPClient(AsyncMPClient):
|
||||
log_stats: bool,
|
||||
client_addresses: Optional[dict[str, str]] = None,
|
||||
client_index: int = 0):
|
||||
|
||||
self.current_wave = 0
|
||||
self.engines_running = False
|
||||
# To route aborts to the correct engine.
|
||||
@@ -1050,3 +1059,50 @@ class DPAsyncMPClient(AsyncMPClient):
|
||||
if not self.resources.engine_dead:
|
||||
await self._send_input(EngineCoreRequestType.ABORT, request_ids,
|
||||
engine)
|
||||
|
||||
|
||||
class RayDPClient(DPAsyncMPClient):
|
||||
"""
|
||||
Ray-based client for multi-proc, multi-engine (data parallel)
|
||||
EngineCore.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vllm_config: VllmConfig,
|
||||
executor_class: type[Executor],
|
||||
log_stats: bool,
|
||||
client_addresses: Optional[dict[str, str]] = None,
|
||||
client_index: int = 0,
|
||||
):
|
||||
super().__init__(vllm_config, executor_class, log_stats,
|
||||
client_addresses, client_index)
|
||||
|
||||
def _init_engines_direct(self, vllm_config: VllmConfig, local_only: bool,
|
||||
local_start_index: int, input_address: str,
|
||||
output_address: str,
|
||||
executor_class: type[Executor], log_stats: bool):
|
||||
"""Self-contained client mode, launch engine and coordinator process
|
||||
as needed."""
|
||||
|
||||
parallel_config = vllm_config.parallel_config
|
||||
assert parallel_config.data_parallel_rank == 0
|
||||
assert local_start_index == 0
|
||||
|
||||
addresses = EngineZmqAddresses(
|
||||
inputs=[input_address],
|
||||
outputs=[output_address],
|
||||
)
|
||||
|
||||
if len(self.core_engines) > 1:
|
||||
coordinator = DPCoordinator(parallel_config)
|
||||
self.resources.coordinator = coordinator
|
||||
addresses.coordinator_input, addresses.coordinator_output = (
|
||||
coordinator.get_engine_socket_addresses())
|
||||
|
||||
# Start all engines.
|
||||
self.resources.engine_manager = CoreEngineActorManager(
|
||||
vllm_config=vllm_config,
|
||||
addresses=addresses,
|
||||
executor_class=executor_class,
|
||||
log_stats=log_stats)
|
||||
|
||||
Reference in New Issue
Block a user