[warmup] overlap renderer warmup and engine core initialization (#54557)

Signed-off-by: Andy Xie <andy.xning@gmail.com>
Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
This commit is contained in:
Ning Xie
2026-09-03 19:44:21 +00:00
committed by GitHub
co-authored by Cyrus Leung
parent fc8f10792c
commit 2a336d8239
5 changed files with 561 additions and 33 deletions
+117 -24
View File
@@ -5,7 +5,7 @@ import time
import weakref
from abc import ABC, abstractmethod
from collections.abc import Mapping, Sequence
from concurrent.futures import ThreadPoolExecutor
from concurrent.futures import Future, ThreadPoolExecutor
from contextlib import ExitStack
from functools import cached_property
from typing import TYPE_CHECKING, Any, Generic, overload
@@ -85,6 +85,26 @@ class BaseRenderer(ABC, Generic[_T]):
self.tokenizer = tokenizer
# Background multimodal warmup task (a future returned by
# _mm_executor) so it overlaps engine-core initialization while
# staying serialized with the serving path through the same
# single-worker executor. None when no background warmup has been
# launched (or it has already been joined). Routing the warmup through
# _mm_executor (max_workers=1) guarantees it can never run
# concurrently with a serving request inside the numba workqueue
# parallel region, which would otherwise hit numba's
# "Concurrent access has been detected" fatal abort (see
# vllm/utils/jit_monitor.py, which forces the workqueue layer).
# Joined by the first of reset_mm_cache / warmup / shutdown so the
# mm_processor_cache is also never touched concurrently by the warmup
# and the serving path.
self._mm_warmup_future: Future[None] | None = None
# Whether warmup_mm has already run. Survives _join_mm_warmup (which
# clears the thread pointer), so warmup() can tell that the background
# warmup already ran even after reset_mm_cache joined it first, and
# avoid running warmup_mm a second time.
self._mm_warmup_done: bool = False
# Thread pool executor for blocking tokenizer operations. The
# multimodal processor receives a deep-copied tokenizer (see #36557)
# so it is safe to run tokenization and MM preprocessing concurrently.
@@ -243,12 +263,97 @@ class BaseRenderer(ABC, Generic[_T]):
elapsed = time.perf_counter() - start_time
logger.info("%s warmup completed in %.3fs", log_prefix, elapsed)
def warmup_mm(self) -> None:
# Run only the multimodal warmup (mm + readonly mm), serially.
# Idempotent: runs at most once. A background warmup
# (start_mm_warmup_in_background) may already have run it, and
# reset_mm_cache may have joined that background warmup task before
# warmup() reaches this point — the _mm_warmup_done flag survives the
# join (the future pointer does not), so warmup() can still tell it
# already ran.
if self._mm_warmup_done:
return
try:
# prevent MM processor hangs
with set_default_torch_num_threads(1):
if self.mm_processor:
try:
logger.debug("Warming up multi-modal processing...")
self._warmup_mm_processor(
self.mm_processor,
log_prefix="Multi-modal",
)
except Exception:
logger.warning("Multi-modal warmup failed")
finally:
self.clear_mm_cache()
if self._readonly_mm_processor is not None:
try:
logger.debug("Warming up readonly multi-modal processing...")
self._warmup_mm_processor(
self._readonly_mm_processor,
log_prefix="Readonly multi-modal",
)
except Exception:
logger.warning("Readonly multi-modal warmup failed")
finally:
self._clear_processor_cache(self._readonly_mm_processor)
finally:
self._mm_warmup_done = True
def start_mm_warmup_in_background(self) -> None:
# Launch the multimodal warmup as a task on the single-worker
# _mm_executor so it overlaps with engine-core initialization
# (model loading) while staying serialized with the serving path.
#
# Why the same executor as serving (not a separate Thread):
# _process_multimodal runs on _mm_executor (max_workers=1) and
# enters the numba workqueue parallel region (e.g. Kimi K2.5 vision
# preprocessing via kimi_k25_vision_fused). A separate warmup Thread
# would run concurrently with that single serving worker, so both
# threads would enter the numba workqueue parallel region at the same
# time and trip numba's "Concurrent access has been detected" ->
# "Fatal Python error: Aborted" (see vllm/utils/jit_monitor.py, which
# forces the non-threadsafe workqueue layer). Submitting the warmup to
# the same single-worker executor makes warmup and serving
# physically incapable of overlapping on the numba threading layer,
# while still overlapping at the process level with the (forked)
# engine-core init in the main thread.
#
# The warmup task is joined by the first of: reset_mm_cache, warmup, or
# shutdown. This also guarantees the frontend mm_processor_cache —
# which is not designed for concurrent access (its get_and_update must
# stay ordered with the engine-core cache) — is never touched
# concurrently by the warmup and the serving path.
if self.mm_processor is None and self._readonly_mm_processor is None:
return # text-only model: nothing to warm up
if self._mm_warmup_future is not None:
return # already launched
self._mm_warmup_future = self._mm_executor.submit(self.warmup_mm)
def _join_mm_warmup(self) -> None:
# Wait for the background MM warmup task to finish, if one was started.
# Called from the main / event-loop thread (reset_mm_cache, warmup,
# shutdown) — never from inside the _mm_executor worker — so
# waiting on the future cannot deadlock.
future = getattr(self, "_mm_warmup_future", None)
if future is not None:
future.result()
self._mm_warmup_future = None
def warmup(self, chat_params: ChatParams) -> None:
"""
Warm up this renderer to avoid first-request latency.
For chat requests:
- Jinja2 template compilation
If a background multimodal warmup was launched (to overlap
engine-core init) it is joined first so it is not still touching the
mm_processor_cache. warmup_mm is idempotent, so the call below is a
no-op if the background warmup — or a prior reset_mm_cache that joined
it — already ran it.
"""
from vllm.entrypoints.chat_utils import ChatTemplateResolutionError
@@ -267,29 +372,12 @@ class BaseRenderer(ABC, Generic[_T]):
except Exception:
logger.warning("Chat template warmup failed", exc_info=True)
if self.mm_processor:
try:
logger.debug("Warming up multi-modal processing...")
self._warmup_mm_processor(
self.mm_processor,
log_prefix="Multi-modal",
)
except Exception:
logger.warning("Multi-modal warmup failed")
finally:
self.clear_mm_cache()
if self._readonly_mm_processor is not None:
try:
logger.debug("Warming up readonly multi-modal processing...")
self._warmup_mm_processor(
self._readonly_mm_processor,
log_prefix="Readonly multi-modal",
)
except Exception:
logger.warning("Readonly multi-modal warmup failed")
finally:
self._clear_processor_cache(self._readonly_mm_processor)
if self._mm_warmup_future is not None:
# Join a background MM warmup launched to overlap engine-core init.
self._join_mm_warmup()
# Idempotent: no-op if the background warmup (or a prior reset_mm_cache)
# already ran the MM warmup.
self.warmup_mm()
async def clear_mm_cache_async(self) -> None:
"""Serialize clear_mm_cache through the multimodal executor to avoid
@@ -297,6 +385,11 @@ class BaseRenderer(ABC, Generic[_T]):
await self._clear_mm_cache_async()
def shutdown(self) -> None:
# Wait for any background MM warmup to finish first: it runs on the
# _mm_executor and touches the mm_processor_cache, so it must be
# quiescent before _resources.close() tears down the executors and
# the cache (both are registered as resources in __init__).
self._join_mm_warmup()
self._resources.close()
def get_bos_token_id(self) -> int | None:
+8
View File
@@ -153,6 +153,10 @@ class AsyncLLM(EngineClient):
)
# EngineCore (starts the engine in background process).
# Hand the renderer to the client so it can start the frontend MM
# warmup only after engine-core fork (the why is in
# BaseRenderer.start_mm_warmup_in_background). The warmup is joined
# by reset_mm_cache / warmup / shutdown.
self.engine_core = EngineCoreClient.make_async_mp_client(
vllm_config=vllm_config,
executor_class=executor_class,
@@ -160,6 +164,7 @@ class AsyncLLM(EngineClient):
client_addresses=client_addresses,
client_count=client_count,
client_index=client_index,
renderer=renderer,
)
# Loggers.
@@ -1023,6 +1028,9 @@ class AsyncLLM(EngineClient):
await asyncio.gather(*coros)
async def reset_mm_cache(self) -> None:
# Join the background MM warmup first: the mm_processor_cache is not
# safe for concurrent access with its apply/clear.
self.renderer._join_mm_warmup()
await self.renderer.clear_mm_cache_async()
await self.engine_core.reset_mm_cache_async()
+82 -8
View File
@@ -26,6 +26,7 @@ from vllm.config import VllmConfig
from vllm.envs import VLLM_ENGINE_READY_TIMEOUT_S
from vllm.logger import init_logger
from vllm.lora.request import LoRARequest
from vllm.renderers import BaseRenderer
from vllm.tasks import SupportedTask
from vllm.tracing import instrument
from vllm.utils.async_utils import in_loop
@@ -93,7 +94,15 @@ class EngineCoreClient(ABC):
vllm_config: VllmConfig,
executor_class: type[Executor],
log_stats: bool,
renderer: BaseRenderer | None = None,
) -> "EngineCoreClient":
# renderer is passed through to the multiprocess clients, which start
# the frontend MM warmup (renderer.start_mm_warmup_in_background) once
# the engine-core processes have been forked, so warmup overlaps
# engine-core load without a live thread at fork() time. In-process
# clients do not take a renderer: the engine core is built in this
# process, so there is no other process to overlap with and the MM
# warmup runs synchronously inside renderer.warmup() instead.
# TODO: support this for debugging purposes.
if asyncio_mode and not multiprocess_mode:
raise NotImplementedError(
@@ -103,11 +112,19 @@ class EngineCoreClient(ABC):
if multiprocess_mode and asyncio_mode:
return EngineCoreClient.make_async_mp_client(
vllm_config, executor_class, log_stats
vllm_config,
executor_class,
log_stats,
renderer=renderer,
)
if multiprocess_mode and not asyncio_mode:
return SyncMPClient(vllm_config, executor_class, log_stats)
return SyncMPClient(
vllm_config,
executor_class,
log_stats,
renderer=renderer,
)
return InprocClient(vllm_config, executor_class, log_stats)
@@ -120,6 +137,7 @@ class EngineCoreClient(ABC):
client_addresses: dict[str, Any] | None = None,
client_count: int = 1,
client_index: int = 0,
renderer: BaseRenderer | None = None,
) -> "AsyncMPClient":
parallel_config = vllm_config.parallel_config
client_args = (
@@ -133,10 +151,19 @@ class EngineCoreClient(ABC):
if parallel_config.data_parallel_size > 1:
if parallel_config.data_parallel_external_lb:
# External load balancer - client per DP rank.
return DPAsyncMPClient(*client_args)
return DPAsyncMPClient(
*client_args,
renderer=renderer,
)
# Internal load balancer - client balances to all DP ranks.
return DPLBAsyncMPClient(*client_args)
return AsyncMPClient(*client_args)
return DPLBAsyncMPClient(
*client_args,
renderer=renderer,
)
return AsyncMPClient(
*client_args,
renderer=renderer,
)
@abstractmethod
def shutdown(self, timeout: float | None = None) -> None: ...
@@ -313,8 +340,23 @@ class InprocClient(EngineCoreClient):
* pulls EngineCoreOutputs by stepping the EngineCore
"""
def __init__(self, *args, **kwargs):
self.engine_core = EngineCore(*args, **kwargs)
# Takes no renderer: this class has no _start_mm_warmup (only MPClient
# does), so a renderer would never be used here. EngineCore is built in
# this process and MM warmup runs synchronously inside renderer.warmup().
def __init__(
self,
vllm_config: VllmConfig,
executor_class: type[Executor],
log_stats: bool,
*,
executor_fail_callback: Callable | None = None,
):
self.engine_core = EngineCore(
vllm_config,
executor_class,
log_stats,
executor_fail_callback=executor_fail_callback,
)
def get_output(self) -> EngineCoreOutputs:
outputs, model_executed = self.engine_core.step_fn()
@@ -520,8 +562,10 @@ class MPClient(EngineCoreClient):
executor_class: type[Executor],
log_stats: bool,
client_addresses: dict[str, Any] | None = None,
renderer: BaseRenderer | None = None,
):
self.vllm_config = vllm_config
self._renderer: BaseRenderer | None = renderer
# ZMQ setup.
sync_ctx = zmq.Context(io_threads=2)
@@ -583,6 +627,9 @@ class MPClient(EngineCoreClient):
)
finally:
actual_address_pipe.close()
# Engines are managed externally: this process does not fork
# them, so there is no fork race; start the MM warmup now.
self._start_mm_warmup()
else:
# Engines are managed by this client.
addresses = get_engine_zmq_addresses(vllm_config)
@@ -614,6 +661,14 @@ class MPClient(EngineCoreClient):
coordinator = engine_launch.coordinator
addresses = engine_launch.addresses
tensor_queue = engine_launch.tensor_queue
# Engine-core processes have now all been forked/started
# (CoreEngineProcManager.proc.start()). It is now safe to
# launch the frontend background MM warmup: it must not
# run while fork() is in flight (a live thread holding a
# lock would deadlock the forked child), but the
# engine-core model load (minutes) that follows is exactly
# what the warmup should overlap with.
self._start_mm_warmup()
self.stats_update_address = addresses.frontend_stats_publish_address
if coordinator is not None:
@@ -708,6 +763,14 @@ class MPClient(EngineCoreClient):
def dp_engines_running(self) -> bool:
return self.engines_running
def _start_mm_warmup(self) -> None:
# Called once the engine-core process(es) have been created (forked or
# externally managed). Overlap the frontend MM warmup with the
# engine-core model load. This is a no-op when no renderer was passed
# (e.g. text-only serving or tests).
if self._renderer is not None:
self._renderer.start_mm_warmup_in_background()
def start_engine_core_monitor(self):
"""Start a monitor thread for engine core processes."""
engine_manager = self.resources.engine_manager
@@ -808,13 +871,18 @@ class SyncMPClient(MPClient):
@instrument(span_name="SyncMPClient init")
def __init__(
self, vllm_config: VllmConfig, executor_class: type[Executor], log_stats: bool
self,
vllm_config: VllmConfig,
executor_class: type[Executor],
log_stats: bool,
renderer: BaseRenderer | None = None,
):
super().__init__(
asyncio_mode=False,
vllm_config=vllm_config,
executor_class=executor_class,
log_stats=log_stats,
renderer=renderer,
)
self.is_dp = self.vllm_config.parallel_config.data_parallel_size > 1
@@ -987,6 +1055,7 @@ class AsyncMPClient(MPClient):
client_addresses: dict[str, Any] | None = None,
client_count: int = 1,
client_index: int = 0,
renderer: BaseRenderer | None = None,
):
super().__init__(
asyncio_mode=True,
@@ -994,6 +1063,7 @@ class AsyncMPClient(MPClient):
executor_class=executor_class,
log_stats=log_stats,
client_addresses=client_addresses,
renderer=renderer,
)
self.client_count = client_count
@@ -1262,6 +1332,7 @@ class DPAsyncMPClient(AsyncMPClient):
client_addresses: dict[str, Any] | None = None,
client_count: int = 1,
client_index: int = 0,
renderer: BaseRenderer | None = None,
):
self.current_wave = 0
@@ -1272,6 +1343,7 @@ class DPAsyncMPClient(AsyncMPClient):
client_addresses,
client_count,
client_index,
renderer,
)
# List of [waiting, running, kv_cache_usage] per engine.
@@ -1444,6 +1516,7 @@ class DPLBAsyncMPClient(DPAsyncMPClient):
client_addresses: dict[str, Any] | None = None,
client_count: int = 1,
client_index: int = 0,
renderer: BaseRenderer | None = None,
):
self.client_count = client_count
@@ -1460,6 +1533,7 @@ class DPLBAsyncMPClient(DPAsyncMPClient):
client_addresses,
client_count,
client_index,
renderer,
)
assert len(self.core_engines) > 1
+8
View File
@@ -102,12 +102,17 @@ class LLMEngine:
)
# EngineCore (gets EngineCoreRequests and gives EngineCoreOutputs)
# Hand the renderer to the client. In multiprocess mode the client
# starts the MM warmup only after engine-core fork (the why is in
# BaseRenderer.start_mm_warmup_in_background); InprocClient takes no
# renderer, so MM warmup stays inside renderer.warmup() there.
self.engine_core = EngineCoreClient.make_client(
multiprocess_mode=multiprocess_mode,
asyncio_mode=False,
vllm_config=vllm_config,
executor_class=executor_class,
log_stats=self.log_stats,
renderer=renderer,
)
self.logger_manager: StatLoggerManager | None = None
@@ -343,6 +348,9 @@ class LLMEngine:
self.engine_core.profile(False)
def reset_mm_cache(self):
# Join the background MM warmup first: the mm_processor_cache is not
# safe for concurrent access with its apply/clear.
self.renderer._join_mm_warmup()
self.renderer.clear_mm_cache()
self.engine_core.reset_mm_cache()