[warmup] overlap renderer warmup and engine core initialization (#52764)

Signed-off-by: Andy Xie <andy.xning@gmail.com>
This commit is contained in:
Ning Xie
2026-08-27 11:42:08 +08:00
committed by GitHub
parent f8d38fbc87
commit ca90b9e7d4
4 changed files with 345 additions and 24 deletions
+195
View File
@@ -6,11 +6,18 @@ These tests exercise:
- Zero-limit modalities are filtered from mm_counts passed to - Zero-limit modalities are filtered from mm_counts passed to
get_dummy_processor_inputs (e.g. --limit-mm-per-prompt image=0 ...) get_dummy_processor_inputs (e.g. --limit-mm-per-prompt image=0 ...)
- MM warmup is skipped entirely when mm_processor is None - MM warmup is skipped entirely when mm_processor is None
- The multimodal warmup is launched as a task on the single-worker
``_mm_executor`` to overlap engine-core init (future lifecycle, join by
warmup/reset/shutdown, no double-run). Routing it through the same
executor that serves ``_process_multimodal`` keeps the numba workqueue
parallel region single-threaded, avoiding the "Concurrent access has been
detected" fatal abort when a request arrives during warmup.
No model weights are required: warmup() is called directly on a MagicMock No model weights are required: warmup() is called directly on a MagicMock
that acts as the renderer instance. that acts as the renderer instance.
""" """
from concurrent.futures import Future, ThreadPoolExecutor
from unittest.mock import MagicMock, patch from unittest.mock import MagicMock, patch
from vllm.renderers.base import BaseRenderer from vllm.renderers.base import BaseRenderer
@@ -39,6 +46,18 @@ def _make_renderer_mock(mm_limits: dict[str, int]) -> MagicMock:
renderer, BaseRenderer renderer, BaseRenderer
) )
renderer._clear_processor_cache = BaseRenderer._clear_processor_cache renderer._clear_processor_cache = BaseRenderer._clear_processor_cache
renderer.warmup_mm = BaseRenderer.warmup_mm.__get__(renderer, BaseRenderer)
renderer.start_mm_warmup_in_background = (
BaseRenderer.start_mm_warmup_in_background.__get__(renderer, BaseRenderer)
)
renderer._join_mm_warmup = BaseRenderer._join_mm_warmup.__get__(
renderer, BaseRenderer
)
renderer.shutdown = BaseRenderer.shutdown.__get__(renderer, BaseRenderer)
# No background warmup launched by default; warmup() takes the inline path.
renderer._mm_warmup_future = None
# MM warmup has not run yet; warmup_mm must actually execute on the mock.
renderer._mm_warmup_done = False
renderer.clear_mm_cache = MagicMock() renderer.clear_mm_cache = MagicMock()
renderer.model_config.max_model_len = 128 renderer.model_config.max_model_len = 128
renderer.model_config.get_multimodal_config.return_value.limit_per_prompt = {} renderer.model_config.get_multimodal_config.return_value.limit_per_prompt = {}
@@ -133,3 +152,179 @@ class TestReadonlyMmWarmup:
readonly_mm_processor.apply.assert_called_once() readonly_mm_processor.apply.assert_called_once()
readonly_mm_processor.cache.clear_cache.assert_called_once() readonly_mm_processor.cache.clear_cache.assert_called_once()
class TestChatWarmupDispatched:
"""The chat-template warmup task must always run."""
def test_render_chat_called_with_warmup_message(self):
renderer = _make_renderer_mock({"image": 1})
with patch("vllm.multimodal.processing.TimingContext", autospec=True):
BaseRenderer.warmup(renderer, ChatParams())
renderer.render_chat.assert_called_once()
class TestWarmupFaultIsolation:
"""A failure during a multimodal processor warmup is caught so it does not
abort the remaining warmup steps; warmup itself must not raise."""
def test_chat_failure_does_not_abort_mm_warmup(self):
renderer = _make_renderer_mock({"image": 1})
renderer.render_chat.side_effect = RuntimeError("chat boom")
with patch("vllm.multimodal.processing.TimingContext", autospec=True):
BaseRenderer.warmup(renderer, ChatParams()) # must not raise
renderer.mm_processor.apply.assert_called_once()
def test_mm_failure_does_not_abort_readonly_warmup(self):
renderer = _make_renderer_mock({"image": 1})
readonly_mm_processor = MagicMock()
readonly_mm_processor.info.allowed_mm_limits = {"image": 1}
renderer._readonly_mm_processor = readonly_mm_processor
# main processor warmup blows up before apply()
renderer.mm_processor.dummy_inputs.get_dummy_processor_inputs.side_effect = (
RuntimeError("mm boom")
)
with patch("vllm.multimodal.processing.TimingContext", autospec=True):
BaseRenderer.warmup(renderer, ChatParams()) # must not raise
readonly_mm_processor.apply.assert_called_once()
# cache is still cleared in the failed task's finally
renderer.clear_mm_cache.assert_called_once()
class TestBackgroundMmWarmup:
"""The multimodal warmup is launched as a task on the single-worker
``_mm_executor`` so it overlaps engine-core init (fork) while staying
serialized with the serving path (``_process_multimodal`` runs on the same
executor). warmup()/reset_mm_cache/shutdown join it. This also fixes the
numba workqueue "Concurrent access has been detected" abort: warmup and a
concurrent serving request can no longer both enter the numba parallel
region at once."""
def _make_bg_renderer(self, mm_limits: dict[str, int]):
"""A renderer mock with a real single-worker executor so submit()
returns a real Future and warmup_mm actually runs (in the worker)."""
renderer = _make_renderer_mock(mm_limits)
renderer._mm_executor = ThreadPoolExecutor(max_workers=1)
return renderer
def test_start_submits_future_to_mm_executor(self):
renderer = self._make_bg_renderer({"image": 1})
try:
with patch("vllm.multimodal.processing.TimingContext", autospec=True):
# Spy on the real executor's submit to confirm the warmup is
# dispatched through _mm_executor (not a separate Thread).
with patch.object(
renderer._mm_executor, "submit", wraps=renderer._mm_executor.submit
) as spy:
renderer.start_mm_warmup_in_background()
assert isinstance(renderer._mm_warmup_future, Future)
assert spy.called
renderer._join_mm_warmup()
assert renderer._mm_warmup_future is None
renderer.mm_processor.apply.assert_called_once()
finally:
renderer._mm_executor.shutdown(wait=True)
def test_start_noop_for_text_only_model(self):
renderer = _make_renderer_mock({})
renderer.mm_processor = None
# _readonly_mm_processor is already None
renderer.start_mm_warmup_in_background()
assert renderer._mm_warmup_future is None
def test_start_is_run_at_most_once(self):
renderer = self._make_bg_renderer({"image": 1})
try:
with patch("vllm.multimodal.processing.TimingContext", autospec=True):
renderer.start_mm_warmup_in_background()
first = renderer._mm_warmup_future
renderer.start_mm_warmup_in_background() # must not spawn a second
assert renderer._mm_warmup_future is first
renderer._join_mm_warmup()
finally:
renderer._mm_executor.shutdown(wait=True)
def test_warmup_joins_background_and_does_not_rerun_mm(self):
"""When a background MM warmup is in flight, warmup() must join it and
run only the chat warmup — the MM warmup must not run twice."""
renderer = self._make_bg_renderer({"image": 1})
try:
with patch("vllm.multimodal.processing.TimingContext", autospec=True):
renderer.start_mm_warmup_in_background()
BaseRenderer.warmup(renderer, ChatParams())
# MM apply called exactly once (by the background warmup task).
renderer.mm_processor.apply.assert_called_once()
# Chat warmup ran exactly once.
renderer.render_chat.assert_called_once()
# Background task has been joined and cleared.
assert renderer._mm_warmup_future is None
finally:
renderer._mm_executor.shutdown(wait=True)
def test_warmup_does_not_rerun_mm_after_reset_joins_background(self):
"""Regression: reset_mm_cache joins the background warmup before
warmup() runs (it clears _mm_warmup_future). warmup() must still not
re-run the MM warmup — the _mm_warmup_done flag survives the join."""
renderer = self._make_bg_renderer({"image": 1})
try:
with patch("vllm.multimodal.processing.TimingContext", autospec=True):
renderer.start_mm_warmup_in_background()
# Simulate reset_mm_cache joining the background warmup first
# (clears _mm_warmup_future, sets _mm_warmup_done via the join).
renderer._join_mm_warmup()
assert renderer._mm_warmup_future is None # joined & cleared
BaseRenderer.warmup(renderer, ChatParams())
# MM apply called exactly once (by the background warmup task);
# warmup() did not re-run warmup_mm despite the future being cleared.
renderer.mm_processor.apply.assert_called_once()
renderer.render_chat.assert_called_once()
finally:
renderer._mm_executor.shutdown(wait=True)
def test_warmup_mm_runs_at_most_once(self):
"""Direct repeated calls to warmup_mm run the MM warmup only once."""
renderer = _make_renderer_mock({"image": 1})
with patch("vllm.multimodal.processing.TimingContext", autospec=True):
renderer.warmup_mm()
renderer.warmup_mm() # second call must be a no-op
renderer.mm_processor.apply.assert_called_once()
def test_join_mm_warmup_blocks_until_apply_done(self):
renderer = self._make_bg_renderer({"image": 1})
try:
with patch("vllm.multimodal.processing.TimingContext", autospec=True):
renderer.start_mm_warmup_in_background()
renderer._join_mm_warmup()
assert renderer._mm_warmup_future is None
renderer.mm_processor.apply.assert_called_once()
finally:
renderer._mm_executor.shutdown(wait=True)
def test_shutdown_joins_background_warmup(self):
"""shutdown() must join the background warmup before closing caches,
so the mm_processor_cache is never touched concurrently."""
renderer = self._make_bg_renderer({"image": 1})
with patch("vllm.multimodal.processing.TimingContext", autospec=True):
renderer.start_mm_warmup_in_background()
future = renderer._mm_warmup_future
renderer.shutdown()
assert renderer._mm_warmup_future is None
# The background warmup was allowed to complete (apply ran) and the
# future is done after shutdown joined it.
assert future.done()
renderer.mm_processor.apply.assert_called_once()
+116 -24
View File
@@ -4,7 +4,7 @@ import asyncio
import time import time
from abc import ABC, abstractmethod from abc import ABC, abstractmethod
from collections.abc import Mapping, Sequence from collections.abc import Mapping, Sequence
from concurrent.futures import Executor, ThreadPoolExecutor from concurrent.futures import Executor, Future, ThreadPoolExecutor
from functools import cached_property from functools import cached_property
from typing import TYPE_CHECKING, Any, Generic, overload from typing import TYPE_CHECKING, Any, Generic, overload
@@ -79,6 +79,26 @@ class BaseRenderer(ABC, Generic[_T]):
self.tokenizer = tokenizer self.tokenizer = tokenizer
# Background multimodal warmup task (a future returned by
# ``_mm_executor``) so it overlaps engine-core initialization while
# staying serialized with the serving path through the same
# single-worker executor. None when no background warmup has been
# launched (or it has already been joined). Routing the warmup through
# ``_mm_executor`` (max_workers=1) guarantees it can never run
# concurrently with a serving request inside the numba workqueue
# parallel region, which would otherwise hit numba's
# "Concurrent access has been detected" fatal abort (see
# vllm/utils/jit_monitor.py, which forces the workqueue layer).
# Joined by the first of reset_mm_cache / warmup / shutdown so the
# mm_processor_cache is also never touched concurrently by the warmup
# and the serving path.
self._mm_warmup_future: Future[None] | None = None
# Whether warmup_mm has already run. Survives _join_mm_warmup (which
# clears the thread pointer), so warmup() can tell that the background
# warmup already ran even after reset_mm_cache joined it first, and
# avoid running warmup_mm a second time.
self._mm_warmup_done: bool = False
# Thread pool executor for blocking tokenizer operations. The # Thread pool executor for blocking tokenizer operations. The
# multimodal processor receives a deep-copied tokenizer (see #36557) # multimodal processor receives a deep-copied tokenizer (see #36557)
# so it is safe to run tokenization and MM preprocessing concurrently. # so it is safe to run tokenization and MM preprocessing concurrently.
@@ -234,12 +254,97 @@ class BaseRenderer(ABC, Generic[_T]):
elapsed = time.perf_counter() - start_time elapsed = time.perf_counter() - start_time
logger.info("%s warmup completed in %.3fs", log_prefix, elapsed) logger.info("%s warmup completed in %.3fs", log_prefix, elapsed)
def warmup_mm(self) -> None:
# Run only the multimodal warmup (mm + readonly mm), serially.
# Idempotent: runs at most once. A background warmup
# (start_mm_warmup_in_background) may already have run it, and
# reset_mm_cache may have joined that background warmup task before
# warmup() reaches this point — the _mm_warmup_done flag survives the
# join (the future pointer does not), so warmup() can still tell it
# already ran.
if self._mm_warmup_done:
return
try:
# prevent MM processor hangs
with set_default_torch_num_threads(1):
if self.mm_processor:
try:
logger.debug("Warming up multi-modal processing...")
self._warmup_mm_processor(
self.mm_processor,
log_prefix="Multi-modal",
)
except Exception:
logger.warning("Multi-modal warmup failed")
finally:
self.clear_mm_cache()
if self._readonly_mm_processor is not None:
try:
logger.debug("Warming up readonly multi-modal processing...")
self._warmup_mm_processor(
self._readonly_mm_processor,
log_prefix="Readonly multi-modal",
)
except Exception:
logger.warning("Readonly multi-modal warmup failed")
finally:
self._clear_processor_cache(self._readonly_mm_processor)
finally:
self._mm_warmup_done = True
def start_mm_warmup_in_background(self) -> None:
# Launch the multimodal warmup as a task on the single-worker
# ``_mm_executor`` so it overlaps with engine-core initialization
# (model loading) while staying serialized with the serving path.
#
# Why the *same* executor as serving (not a separate Thread):
# ``_process_multimodal`` runs on ``_mm_executor`` (max_workers=1) and
# enters the numba workqueue parallel region (e.g. Kimi K2.5 vision
# preprocessing via kimi_k25_vision_fused). A separate warmup Thread
# would run concurrently with that single serving worker, so both
# threads would enter the numba workqueue parallel region at the same
# time and trip numba's "Concurrent access has been detected" ->
# "Fatal Python error: Aborted" (see vllm/utils/jit_monitor.py, which
# forces the non-threadsafe workqueue layer). Submitting the warmup to
# the *same* single-worker executor makes warmup and serving
# physically incapable of overlapping on the numba threading layer,
# while still overlapping at the process level with the (forked)
# engine-core init in the main thread.
#
# The warmup task is joined by the first of: reset_mm_cache, warmup, or
# shutdown. This also guarantees the frontend mm_processor_cache —
# which is not designed for concurrent access (its get_and_update must
# stay ordered with the engine-core cache) — is never touched
# concurrently by the warmup and the serving path.
if self.mm_processor is None and self._readonly_mm_processor is None:
return # text-only model: nothing to warm up
if self._mm_warmup_future is not None:
return # already launched
self._mm_warmup_future = self._mm_executor.submit(self.warmup_mm)
def _join_mm_warmup(self) -> None:
# Wait for the background MM warmup task to finish, if one was started.
# Called from the main / event-loop thread (reset_mm_cache, warmup,
# shutdown) — never from inside the ``_mm_executor`` worker — so
# waiting on the future cannot deadlock.
future = getattr(self, "_mm_warmup_future", None)
if future is not None:
future.result()
self._mm_warmup_future = None
def warmup(self, chat_params: ChatParams) -> None: def warmup(self, chat_params: ChatParams) -> None:
""" """
Warm up this renderer to avoid first-request latency. Warm up this renderer to avoid first-request latency.
For chat requests: For chat requests:
- Jinja2 template compilation - Jinja2 template compilation
If a background multimodal warmup was launched (to overlap
engine-core init) it is joined first so it is not still touching the
mm_processor_cache. warmup_mm is idempotent, so the call below is a
no-op if the background warmup — or a prior reset_mm_cache that joined
it — already ran it.
""" """
from vllm.entrypoints.chat_utils import ChatTemplateResolutionError from vllm.entrypoints.chat_utils import ChatTemplateResolutionError
@@ -258,29 +363,12 @@ class BaseRenderer(ABC, Generic[_T]):
except Exception: except Exception:
logger.warning("Chat template warmup failed", exc_info=True) logger.warning("Chat template warmup failed", exc_info=True)
if self.mm_processor: if self._mm_warmup_future is not None:
try: # Join a background MM warmup launched to overlap engine-core init.
logger.debug("Warming up multi-modal processing...") self._join_mm_warmup()
self._warmup_mm_processor( # Idempotent: no-op if the background warmup (or a prior reset_mm_cache)
self.mm_processor, # already ran the MM warmup.
log_prefix="Multi-modal", self.warmup_mm()
)
except Exception:
logger.warning("Multi-modal warmup failed")
finally:
self.clear_mm_cache()
if self._readonly_mm_processor is not None:
try:
logger.debug("Warming up readonly multi-modal processing...")
self._warmup_mm_processor(
self._readonly_mm_processor,
log_prefix="Readonly multi-modal",
)
except Exception:
logger.warning("Readonly multi-modal warmup failed")
finally:
self._clear_processor_cache(self._readonly_mm_processor)
async def clear_mm_cache_async(self) -> None: async def clear_mm_cache_async(self) -> None:
"""Serialize clear_mm_cache through the multimodal executor to avoid """Serialize clear_mm_cache through the multimodal executor to avoid
@@ -288,6 +376,10 @@ class BaseRenderer(ABC, Generic[_T]):
await self._clear_mm_cache_async() await self._clear_mm_cache_async()
def shutdown(self) -> None: def shutdown(self) -> None:
# Wait for any background MM warmup to finish before closing the
# mm_processor_cache and stopping the executors.
self._join_mm_warmup()
mm_processor_cache = self.mm_processor_cache mm_processor_cache = self.mm_processor_cache
if mm_processor_cache is not None: if mm_processor_cache is not None:
mm_processor_cache.close() mm_processor_cache.close()
+17
View File
@@ -137,6 +137,19 @@ class AsyncLLM(EngineClient):
# Convert EngineInput --> EngineCoreRequest. # Convert EngineInput --> EngineCoreRequest.
self.input_processor = InputProcessor(self.vllm_config, renderer) self.input_processor = InputProcessor(self.vllm_config, renderer)
# Launch the multimodal warmup in the background so it overlaps
# engine-core initialization (model loading, started below). Must be
# after InputProcessor: its MultiModalBudget runs a synchronous MM
# processing pass (get_dummy_mm_inputs -> processor.apply), and the HF
# processor's Numba parallel kernels (e.g. Kimi-K2.5 vision fused) are
# not thread-safe — launching the warmup thread any earlier would run
# it concurrently with that pass and abort. The MM warmup is
# frontend-only (sends no engine_core requests) and only needs the
# renderer. Joined by reset_mm_cache / warmup / shutdown so the
# mm_processor_cache is never touched concurrently by the warmup and
# the serving path.
renderer.start_mm_warmup_in_background()
# Converts EngineCoreOutputs --> RequestOutput. # Converts EngineCoreOutputs --> RequestOutput.
self.output_processor = OutputProcessor( self.output_processor = OutputProcessor(
renderer.tokenizer, renderer.tokenizer,
@@ -956,6 +969,10 @@ class AsyncLLM(EngineClient):
await asyncio.gather(*coros) await asyncio.gather(*coros)
async def reset_mm_cache(self) -> None: async def reset_mm_cache(self) -> None:
# Join any background MM warmup before clearing: the mm_processor_cache
# is not designed for concurrent access, so the warmup's apply/clear
# must have finished before we clear here.
self.renderer._join_mm_warmup()
await self.renderer.clear_mm_cache_async() await self.renderer.clear_mm_cache_async()
await self.engine_core.reset_mm_cache_async() await self.engine_core.reset_mm_cache_async()
+17
View File
@@ -93,6 +93,19 @@ class LLMEngine:
# Convert EngineInput --> EngineCoreRequest. # Convert EngineInput --> EngineCoreRequest.
self.input_processor = InputProcessor(self.vllm_config, renderer) self.input_processor = InputProcessor(self.vllm_config, renderer)
# Launch the multimodal warmup in the background so it overlaps
# engine-core initialization (model loading, started below). Must be
# after InputProcessor: its MultiModalBudget runs a synchronous MM
# processing pass (get_dummy_mm_inputs -> processor.apply), and the HF
# processor's Numba parallel kernels (e.g. Kimi-K2.5 vision fused) are
# not thread-safe — launching the warmup thread any earlier would run
# it concurrently with that pass and abort. The MM warmup is
# frontend-only (sends no engine_core requests) and only needs the
# renderer. Joined by reset_mm_cache / warmup / shutdown so the
# mm_processor_cache is never touched concurrently by the warmup and
# the serving path.
renderer.start_mm_warmup_in_background()
# Converts EngineCoreOutputs --> RequestOutput. # Converts EngineCoreOutputs --> RequestOutput.
self.output_processor = OutputProcessor( self.output_processor = OutputProcessor(
renderer.tokenizer, renderer.tokenizer,
@@ -343,6 +356,10 @@ class LLMEngine:
self.engine_core.profile(False) self.engine_core.profile(False)
def reset_mm_cache(self): def reset_mm_cache(self):
# Join any background MM warmup before clearing: the mm_processor_cache
# is not designed for concurrent access, so the warmup's apply/clear
# must have finished before we clear here.
self.renderer._join_mm_warmup()
self.renderer.clear_mm_cache() self.renderer.clear_mm_cache()
self.engine_core.reset_mm_cache() self.engine_core.reset_mm_cache()