[warmup] overlap renderer warmup and engine core initialization (#54557)
Signed-off-by: Andy Xie <andy.xning@gmail.com> Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
This commit is contained in:
@@ -26,6 +26,7 @@ from vllm.config import VllmConfig
|
||||
from vllm.envs import VLLM_ENGINE_READY_TIMEOUT_S
|
||||
from vllm.logger import init_logger
|
||||
from vllm.lora.request import LoRARequest
|
||||
from vllm.renderers import BaseRenderer
|
||||
from vllm.tasks import SupportedTask
|
||||
from vllm.tracing import instrument
|
||||
from vllm.utils.async_utils import in_loop
|
||||
@@ -93,7 +94,15 @@ class EngineCoreClient(ABC):
|
||||
vllm_config: VllmConfig,
|
||||
executor_class: type[Executor],
|
||||
log_stats: bool,
|
||||
renderer: BaseRenderer | None = None,
|
||||
) -> "EngineCoreClient":
|
||||
# renderer is passed through to the multiprocess clients, which start
|
||||
# the frontend MM warmup (renderer.start_mm_warmup_in_background) once
|
||||
# the engine-core processes have been forked, so warmup overlaps
|
||||
# engine-core load without a live thread at fork() time. In-process
|
||||
# clients do not take a renderer: the engine core is built in this
|
||||
# process, so there is no other process to overlap with and the MM
|
||||
# warmup runs synchronously inside renderer.warmup() instead.
|
||||
# TODO: support this for debugging purposes.
|
||||
if asyncio_mode and not multiprocess_mode:
|
||||
raise NotImplementedError(
|
||||
@@ -103,11 +112,19 @@ class EngineCoreClient(ABC):
|
||||
|
||||
if multiprocess_mode and asyncio_mode:
|
||||
return EngineCoreClient.make_async_mp_client(
|
||||
vllm_config, executor_class, log_stats
|
||||
vllm_config,
|
||||
executor_class,
|
||||
log_stats,
|
||||
renderer=renderer,
|
||||
)
|
||||
|
||||
if multiprocess_mode and not asyncio_mode:
|
||||
return SyncMPClient(vllm_config, executor_class, log_stats)
|
||||
return SyncMPClient(
|
||||
vllm_config,
|
||||
executor_class,
|
||||
log_stats,
|
||||
renderer=renderer,
|
||||
)
|
||||
|
||||
return InprocClient(vllm_config, executor_class, log_stats)
|
||||
|
||||
@@ -120,6 +137,7 @@ class EngineCoreClient(ABC):
|
||||
client_addresses: dict[str, Any] | None = None,
|
||||
client_count: int = 1,
|
||||
client_index: int = 0,
|
||||
renderer: BaseRenderer | None = None,
|
||||
) -> "AsyncMPClient":
|
||||
parallel_config = vllm_config.parallel_config
|
||||
client_args = (
|
||||
@@ -133,10 +151,19 @@ class EngineCoreClient(ABC):
|
||||
if parallel_config.data_parallel_size > 1:
|
||||
if parallel_config.data_parallel_external_lb:
|
||||
# External load balancer - client per DP rank.
|
||||
return DPAsyncMPClient(*client_args)
|
||||
return DPAsyncMPClient(
|
||||
*client_args,
|
||||
renderer=renderer,
|
||||
)
|
||||
# Internal load balancer - client balances to all DP ranks.
|
||||
return DPLBAsyncMPClient(*client_args)
|
||||
return AsyncMPClient(*client_args)
|
||||
return DPLBAsyncMPClient(
|
||||
*client_args,
|
||||
renderer=renderer,
|
||||
)
|
||||
return AsyncMPClient(
|
||||
*client_args,
|
||||
renderer=renderer,
|
||||
)
|
||||
|
||||
@abstractmethod
|
||||
def shutdown(self, timeout: float | None = None) -> None: ...
|
||||
@@ -313,8 +340,23 @@ class InprocClient(EngineCoreClient):
|
||||
* pulls EngineCoreOutputs by stepping the EngineCore
|
||||
"""
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
self.engine_core = EngineCore(*args, **kwargs)
|
||||
# Takes no renderer: this class has no _start_mm_warmup (only MPClient
|
||||
# does), so a renderer would never be used here. EngineCore is built in
|
||||
# this process and MM warmup runs synchronously inside renderer.warmup().
|
||||
def __init__(
|
||||
self,
|
||||
vllm_config: VllmConfig,
|
||||
executor_class: type[Executor],
|
||||
log_stats: bool,
|
||||
*,
|
||||
executor_fail_callback: Callable | None = None,
|
||||
):
|
||||
self.engine_core = EngineCore(
|
||||
vllm_config,
|
||||
executor_class,
|
||||
log_stats,
|
||||
executor_fail_callback=executor_fail_callback,
|
||||
)
|
||||
|
||||
def get_output(self) -> EngineCoreOutputs:
|
||||
outputs, model_executed = self.engine_core.step_fn()
|
||||
@@ -520,8 +562,10 @@ class MPClient(EngineCoreClient):
|
||||
executor_class: type[Executor],
|
||||
log_stats: bool,
|
||||
client_addresses: dict[str, Any] | None = None,
|
||||
renderer: BaseRenderer | None = None,
|
||||
):
|
||||
self.vllm_config = vllm_config
|
||||
self._renderer: BaseRenderer | None = renderer
|
||||
|
||||
# ZMQ setup.
|
||||
sync_ctx = zmq.Context(io_threads=2)
|
||||
@@ -583,6 +627,9 @@ class MPClient(EngineCoreClient):
|
||||
)
|
||||
finally:
|
||||
actual_address_pipe.close()
|
||||
# Engines are managed externally: this process does not fork
|
||||
# them, so there is no fork race; start the MM warmup now.
|
||||
self._start_mm_warmup()
|
||||
else:
|
||||
# Engines are managed by this client.
|
||||
addresses = get_engine_zmq_addresses(vllm_config)
|
||||
@@ -614,6 +661,14 @@ class MPClient(EngineCoreClient):
|
||||
coordinator = engine_launch.coordinator
|
||||
addresses = engine_launch.addresses
|
||||
tensor_queue = engine_launch.tensor_queue
|
||||
# Engine-core processes have now all been forked/started
|
||||
# (CoreEngineProcManager.proc.start()). It is now safe to
|
||||
# launch the frontend background MM warmup: it must not
|
||||
# run while fork() is in flight (a live thread holding a
|
||||
# lock would deadlock the forked child), but the
|
||||
# engine-core model load (minutes) that follows is exactly
|
||||
# what the warmup should overlap with.
|
||||
self._start_mm_warmup()
|
||||
|
||||
self.stats_update_address = addresses.frontend_stats_publish_address
|
||||
if coordinator is not None:
|
||||
@@ -708,6 +763,14 @@ class MPClient(EngineCoreClient):
|
||||
def dp_engines_running(self) -> bool:
|
||||
return self.engines_running
|
||||
|
||||
def _start_mm_warmup(self) -> None:
|
||||
# Called once the engine-core process(es) have been created (forked or
|
||||
# externally managed). Overlap the frontend MM warmup with the
|
||||
# engine-core model load. This is a no-op when no renderer was passed
|
||||
# (e.g. text-only serving or tests).
|
||||
if self._renderer is not None:
|
||||
self._renderer.start_mm_warmup_in_background()
|
||||
|
||||
def start_engine_core_monitor(self):
|
||||
"""Start a monitor thread for engine core processes."""
|
||||
engine_manager = self.resources.engine_manager
|
||||
@@ -808,13 +871,18 @@ class SyncMPClient(MPClient):
|
||||
|
||||
@instrument(span_name="SyncMPClient init")
|
||||
def __init__(
|
||||
self, vllm_config: VllmConfig, executor_class: type[Executor], log_stats: bool
|
||||
self,
|
||||
vllm_config: VllmConfig,
|
||||
executor_class: type[Executor],
|
||||
log_stats: bool,
|
||||
renderer: BaseRenderer | None = None,
|
||||
):
|
||||
super().__init__(
|
||||
asyncio_mode=False,
|
||||
vllm_config=vllm_config,
|
||||
executor_class=executor_class,
|
||||
log_stats=log_stats,
|
||||
renderer=renderer,
|
||||
)
|
||||
|
||||
self.is_dp = self.vllm_config.parallel_config.data_parallel_size > 1
|
||||
@@ -987,6 +1055,7 @@ class AsyncMPClient(MPClient):
|
||||
client_addresses: dict[str, Any] | None = None,
|
||||
client_count: int = 1,
|
||||
client_index: int = 0,
|
||||
renderer: BaseRenderer | None = None,
|
||||
):
|
||||
super().__init__(
|
||||
asyncio_mode=True,
|
||||
@@ -994,6 +1063,7 @@ class AsyncMPClient(MPClient):
|
||||
executor_class=executor_class,
|
||||
log_stats=log_stats,
|
||||
client_addresses=client_addresses,
|
||||
renderer=renderer,
|
||||
)
|
||||
|
||||
self.client_count = client_count
|
||||
@@ -1262,6 +1332,7 @@ class DPAsyncMPClient(AsyncMPClient):
|
||||
client_addresses: dict[str, Any] | None = None,
|
||||
client_count: int = 1,
|
||||
client_index: int = 0,
|
||||
renderer: BaseRenderer | None = None,
|
||||
):
|
||||
self.current_wave = 0
|
||||
|
||||
@@ -1272,6 +1343,7 @@ class DPAsyncMPClient(AsyncMPClient):
|
||||
client_addresses,
|
||||
client_count,
|
||||
client_index,
|
||||
renderer,
|
||||
)
|
||||
|
||||
# List of [waiting, running, kv_cache_usage] per engine.
|
||||
@@ -1444,6 +1516,7 @@ class DPLBAsyncMPClient(DPAsyncMPClient):
|
||||
client_addresses: dict[str, Any] | None = None,
|
||||
client_count: int = 1,
|
||||
client_index: int = 0,
|
||||
renderer: BaseRenderer | None = None,
|
||||
):
|
||||
self.client_count = client_count
|
||||
|
||||
@@ -1460,6 +1533,7 @@ class DPLBAsyncMPClient(DPAsyncMPClient):
|
||||
client_addresses,
|
||||
client_count,
|
||||
client_index,
|
||||
renderer,
|
||||
)
|
||||
|
||||
assert len(self.core_engines) > 1
|
||||
|
||||
Reference in New Issue
Block a user