[Bugfix] Revert renderer warmup overlap to avoid fork deadlock (#54023)
Signed-off-by: Kevin Luu <51931015+khluu@users.noreply.github.com>
This commit is contained in:
@@ -6,18 +6,11 @@ These tests exercise:
|
||||
- Zero-limit modalities are filtered from mm_counts passed to
|
||||
get_dummy_processor_inputs (e.g. --limit-mm-per-prompt image=0 ...)
|
||||
- MM warmup is skipped entirely when mm_processor is None
|
||||
- The multimodal warmup is launched as a task on the single-worker
|
||||
``_mm_executor`` to overlap engine-core init (future lifecycle, join by
|
||||
warmup/reset/shutdown, no double-run). Routing it through the same
|
||||
executor that serves ``_process_multimodal`` keeps the numba workqueue
|
||||
parallel region single-threaded, avoiding the "Concurrent access has been
|
||||
detected" fatal abort when a request arrives during warmup.
|
||||
|
||||
No model weights are required: warmup() is called directly on a MagicMock
|
||||
that acts as the renderer instance.
|
||||
"""
|
||||
|
||||
from concurrent.futures import Future, ThreadPoolExecutor
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from vllm.renderers.base import BaseRenderer
|
||||
@@ -46,18 +39,6 @@ def _make_renderer_mock(mm_limits: dict[str, int]) -> MagicMock:
|
||||
renderer, BaseRenderer
|
||||
)
|
||||
renderer._clear_processor_cache = BaseRenderer._clear_processor_cache
|
||||
renderer.warmup_mm = BaseRenderer.warmup_mm.__get__(renderer, BaseRenderer)
|
||||
renderer.start_mm_warmup_in_background = (
|
||||
BaseRenderer.start_mm_warmup_in_background.__get__(renderer, BaseRenderer)
|
||||
)
|
||||
renderer._join_mm_warmup = BaseRenderer._join_mm_warmup.__get__(
|
||||
renderer, BaseRenderer
|
||||
)
|
||||
renderer.shutdown = BaseRenderer.shutdown.__get__(renderer, BaseRenderer)
|
||||
# No background warmup launched by default; warmup() takes the inline path.
|
||||
renderer._mm_warmup_future = None
|
||||
# MM warmup has not run yet; warmup_mm must actually execute on the mock.
|
||||
renderer._mm_warmup_done = False
|
||||
renderer.clear_mm_cache = MagicMock()
|
||||
renderer.model_config.max_model_len = 128
|
||||
renderer.model_config.get_multimodal_config.return_value.limit_per_prompt = {}
|
||||
@@ -152,179 +133,3 @@ class TestReadonlyMmWarmup:
|
||||
|
||||
readonly_mm_processor.apply.assert_called_once()
|
||||
readonly_mm_processor.cache.clear_cache.assert_called_once()
|
||||
|
||||
|
||||
class TestChatWarmupDispatched:
|
||||
"""The chat-template warmup task must always run."""
|
||||
|
||||
def test_render_chat_called_with_warmup_message(self):
|
||||
renderer = _make_renderer_mock({"image": 1})
|
||||
|
||||
with patch("vllm.multimodal.processing.TimingContext", autospec=True):
|
||||
BaseRenderer.warmup(renderer, ChatParams())
|
||||
|
||||
renderer.render_chat.assert_called_once()
|
||||
|
||||
|
||||
class TestWarmupFaultIsolation:
|
||||
"""A failure during a multimodal processor warmup is caught so it does not
|
||||
abort the remaining warmup steps; warmup itself must not raise."""
|
||||
|
||||
def test_chat_failure_does_not_abort_mm_warmup(self):
|
||||
renderer = _make_renderer_mock({"image": 1})
|
||||
renderer.render_chat.side_effect = RuntimeError("chat boom")
|
||||
|
||||
with patch("vllm.multimodal.processing.TimingContext", autospec=True):
|
||||
BaseRenderer.warmup(renderer, ChatParams()) # must not raise
|
||||
|
||||
renderer.mm_processor.apply.assert_called_once()
|
||||
|
||||
def test_mm_failure_does_not_abort_readonly_warmup(self):
|
||||
renderer = _make_renderer_mock({"image": 1})
|
||||
readonly_mm_processor = MagicMock()
|
||||
readonly_mm_processor.info.allowed_mm_limits = {"image": 1}
|
||||
renderer._readonly_mm_processor = readonly_mm_processor
|
||||
# main processor warmup blows up before apply()
|
||||
renderer.mm_processor.dummy_inputs.get_dummy_processor_inputs.side_effect = (
|
||||
RuntimeError("mm boom")
|
||||
)
|
||||
|
||||
with patch("vllm.multimodal.processing.TimingContext", autospec=True):
|
||||
BaseRenderer.warmup(renderer, ChatParams()) # must not raise
|
||||
|
||||
readonly_mm_processor.apply.assert_called_once()
|
||||
# cache is still cleared in the failed task's finally
|
||||
renderer.clear_mm_cache.assert_called_once()
|
||||
|
||||
|
||||
class TestBackgroundMmWarmup:
|
||||
"""The multimodal warmup is launched as a task on the single-worker
|
||||
``_mm_executor`` so it overlaps engine-core init (fork) while staying
|
||||
serialized with the serving path (``_process_multimodal`` runs on the same
|
||||
executor). warmup()/reset_mm_cache/shutdown join it. This also fixes the
|
||||
numba workqueue "Concurrent access has been detected" abort: warmup and a
|
||||
concurrent serving request can no longer both enter the numba parallel
|
||||
region at once."""
|
||||
|
||||
def _make_bg_renderer(self, mm_limits: dict[str, int]):
|
||||
"""A renderer mock with a real single-worker executor so submit()
|
||||
returns a real Future and warmup_mm actually runs (in the worker)."""
|
||||
renderer = _make_renderer_mock(mm_limits)
|
||||
renderer._mm_executor = ThreadPoolExecutor(max_workers=1)
|
||||
return renderer
|
||||
|
||||
def test_start_submits_future_to_mm_executor(self):
|
||||
renderer = self._make_bg_renderer({"image": 1})
|
||||
try:
|
||||
with patch("vllm.multimodal.processing.TimingContext", autospec=True):
|
||||
# Spy on the real executor's submit to confirm the warmup is
|
||||
# dispatched through _mm_executor (not a separate Thread).
|
||||
with patch.object(
|
||||
renderer._mm_executor, "submit", wraps=renderer._mm_executor.submit
|
||||
) as spy:
|
||||
renderer.start_mm_warmup_in_background()
|
||||
assert isinstance(renderer._mm_warmup_future, Future)
|
||||
assert spy.called
|
||||
renderer._join_mm_warmup()
|
||||
assert renderer._mm_warmup_future is None
|
||||
renderer.mm_processor.apply.assert_called_once()
|
||||
finally:
|
||||
renderer._mm_executor.shutdown(wait=True)
|
||||
|
||||
def test_start_noop_for_text_only_model(self):
|
||||
renderer = _make_renderer_mock({})
|
||||
renderer.mm_processor = None
|
||||
# _readonly_mm_processor is already None
|
||||
|
||||
renderer.start_mm_warmup_in_background()
|
||||
|
||||
assert renderer._mm_warmup_future is None
|
||||
|
||||
def test_start_is_run_at_most_once(self):
|
||||
renderer = self._make_bg_renderer({"image": 1})
|
||||
try:
|
||||
with patch("vllm.multimodal.processing.TimingContext", autospec=True):
|
||||
renderer.start_mm_warmup_in_background()
|
||||
first = renderer._mm_warmup_future
|
||||
renderer.start_mm_warmup_in_background() # must not spawn a second
|
||||
assert renderer._mm_warmup_future is first
|
||||
renderer._join_mm_warmup()
|
||||
finally:
|
||||
renderer._mm_executor.shutdown(wait=True)
|
||||
|
||||
def test_warmup_joins_background_and_does_not_rerun_mm(self):
|
||||
"""When a background MM warmup is in flight, warmup() must join it and
|
||||
run only the chat warmup — the MM warmup must not run twice."""
|
||||
renderer = self._make_bg_renderer({"image": 1})
|
||||
try:
|
||||
with patch("vllm.multimodal.processing.TimingContext", autospec=True):
|
||||
renderer.start_mm_warmup_in_background()
|
||||
BaseRenderer.warmup(renderer, ChatParams())
|
||||
|
||||
# MM apply called exactly once (by the background warmup task).
|
||||
renderer.mm_processor.apply.assert_called_once()
|
||||
# Chat warmup ran exactly once.
|
||||
renderer.render_chat.assert_called_once()
|
||||
# Background task has been joined and cleared.
|
||||
assert renderer._mm_warmup_future is None
|
||||
finally:
|
||||
renderer._mm_executor.shutdown(wait=True)
|
||||
|
||||
def test_warmup_does_not_rerun_mm_after_reset_joins_background(self):
|
||||
"""Regression: reset_mm_cache joins the background warmup before
|
||||
warmup() runs (it clears _mm_warmup_future). warmup() must still not
|
||||
re-run the MM warmup — the _mm_warmup_done flag survives the join."""
|
||||
renderer = self._make_bg_renderer({"image": 1})
|
||||
try:
|
||||
with patch("vllm.multimodal.processing.TimingContext", autospec=True):
|
||||
renderer.start_mm_warmup_in_background()
|
||||
# Simulate reset_mm_cache joining the background warmup first
|
||||
# (clears _mm_warmup_future, sets _mm_warmup_done via the join).
|
||||
renderer._join_mm_warmup()
|
||||
assert renderer._mm_warmup_future is None # joined & cleared
|
||||
BaseRenderer.warmup(renderer, ChatParams())
|
||||
|
||||
# MM apply called exactly once (by the background warmup task);
|
||||
# warmup() did not re-run warmup_mm despite the future being cleared.
|
||||
renderer.mm_processor.apply.assert_called_once()
|
||||
renderer.render_chat.assert_called_once()
|
||||
finally:
|
||||
renderer._mm_executor.shutdown(wait=True)
|
||||
|
||||
def test_warmup_mm_runs_at_most_once(self):
|
||||
"""Direct repeated calls to warmup_mm run the MM warmup only once."""
|
||||
renderer = _make_renderer_mock({"image": 1})
|
||||
|
||||
with patch("vllm.multimodal.processing.TimingContext", autospec=True):
|
||||
renderer.warmup_mm()
|
||||
renderer.warmup_mm() # second call must be a no-op
|
||||
|
||||
renderer.mm_processor.apply.assert_called_once()
|
||||
|
||||
def test_join_mm_warmup_blocks_until_apply_done(self):
|
||||
renderer = self._make_bg_renderer({"image": 1})
|
||||
try:
|
||||
with patch("vllm.multimodal.processing.TimingContext", autospec=True):
|
||||
renderer.start_mm_warmup_in_background()
|
||||
renderer._join_mm_warmup()
|
||||
|
||||
assert renderer._mm_warmup_future is None
|
||||
renderer.mm_processor.apply.assert_called_once()
|
||||
finally:
|
||||
renderer._mm_executor.shutdown(wait=True)
|
||||
|
||||
def test_shutdown_joins_background_warmup(self):
|
||||
"""shutdown() must join the background warmup before closing caches,
|
||||
so the mm_processor_cache is never touched concurrently."""
|
||||
renderer = self._make_bg_renderer({"image": 1})
|
||||
|
||||
with patch("vllm.multimodal.processing.TimingContext", autospec=True):
|
||||
renderer.start_mm_warmup_in_background()
|
||||
future = renderer._mm_warmup_future
|
||||
renderer.shutdown()
|
||||
|
||||
assert renderer._mm_warmup_future is None
|
||||
# The background warmup was allowed to complete (apply ran) and the
|
||||
# future is done after shutdown joined it.
|
||||
assert future.done()
|
||||
renderer.mm_processor.apply.assert_called_once()
|
||||
|
||||
+24
-116
@@ -4,7 +4,7 @@ import asyncio
|
||||
import time
|
||||
from abc import ABC, abstractmethod
|
||||
from collections.abc import Mapping, Sequence
|
||||
from concurrent.futures import Executor, Future, ThreadPoolExecutor
|
||||
from concurrent.futures import Executor, ThreadPoolExecutor
|
||||
from functools import cached_property
|
||||
from typing import TYPE_CHECKING, Any, Generic, overload
|
||||
|
||||
@@ -79,26 +79,6 @@ class BaseRenderer(ABC, Generic[_T]):
|
||||
|
||||
self.tokenizer = tokenizer
|
||||
|
||||
# Background multimodal warmup task (a future returned by
|
||||
# ``_mm_executor``) so it overlaps engine-core initialization while
|
||||
# staying serialized with the serving path through the same
|
||||
# single-worker executor. None when no background warmup has been
|
||||
# launched (or it has already been joined). Routing the warmup through
|
||||
# ``_mm_executor`` (max_workers=1) guarantees it can never run
|
||||
# concurrently with a serving request inside the numba workqueue
|
||||
# parallel region, which would otherwise hit numba's
|
||||
# "Concurrent access has been detected" fatal abort (see
|
||||
# vllm/utils/jit_monitor.py, which forces the workqueue layer).
|
||||
# Joined by the first of reset_mm_cache / warmup / shutdown so the
|
||||
# mm_processor_cache is also never touched concurrently by the warmup
|
||||
# and the serving path.
|
||||
self._mm_warmup_future: Future[None] | None = None
|
||||
# Whether warmup_mm has already run. Survives _join_mm_warmup (which
|
||||
# clears the thread pointer), so warmup() can tell that the background
|
||||
# warmup already ran even after reset_mm_cache joined it first, and
|
||||
# avoid running warmup_mm a second time.
|
||||
self._mm_warmup_done: bool = False
|
||||
|
||||
# Thread pool executor for blocking tokenizer operations. The
|
||||
# multimodal processor receives a deep-copied tokenizer (see #36557)
|
||||
# so it is safe to run tokenization and MM preprocessing concurrently.
|
||||
@@ -254,97 +234,12 @@ class BaseRenderer(ABC, Generic[_T]):
|
||||
elapsed = time.perf_counter() - start_time
|
||||
logger.info("%s warmup completed in %.3fs", log_prefix, elapsed)
|
||||
|
||||
def warmup_mm(self) -> None:
|
||||
# Run only the multimodal warmup (mm + readonly mm), serially.
|
||||
# Idempotent: runs at most once. A background warmup
|
||||
# (start_mm_warmup_in_background) may already have run it, and
|
||||
# reset_mm_cache may have joined that background warmup task before
|
||||
# warmup() reaches this point — the _mm_warmup_done flag survives the
|
||||
# join (the future pointer does not), so warmup() can still tell it
|
||||
# already ran.
|
||||
if self._mm_warmup_done:
|
||||
return
|
||||
try:
|
||||
# prevent MM processor hangs
|
||||
with set_default_torch_num_threads(1):
|
||||
if self.mm_processor:
|
||||
try:
|
||||
logger.debug("Warming up multi-modal processing...")
|
||||
self._warmup_mm_processor(
|
||||
self.mm_processor,
|
||||
log_prefix="Multi-modal",
|
||||
)
|
||||
except Exception:
|
||||
logger.warning("Multi-modal warmup failed")
|
||||
finally:
|
||||
self.clear_mm_cache()
|
||||
|
||||
if self._readonly_mm_processor is not None:
|
||||
try:
|
||||
logger.debug("Warming up readonly multi-modal processing...")
|
||||
self._warmup_mm_processor(
|
||||
self._readonly_mm_processor,
|
||||
log_prefix="Readonly multi-modal",
|
||||
)
|
||||
except Exception:
|
||||
logger.warning("Readonly multi-modal warmup failed")
|
||||
finally:
|
||||
self._clear_processor_cache(self._readonly_mm_processor)
|
||||
finally:
|
||||
self._mm_warmup_done = True
|
||||
|
||||
def start_mm_warmup_in_background(self) -> None:
|
||||
# Launch the multimodal warmup as a task on the single-worker
|
||||
# ``_mm_executor`` so it overlaps with engine-core initialization
|
||||
# (model loading) while staying serialized with the serving path.
|
||||
#
|
||||
# Why the *same* executor as serving (not a separate Thread):
|
||||
# ``_process_multimodal`` runs on ``_mm_executor`` (max_workers=1) and
|
||||
# enters the numba workqueue parallel region (e.g. Kimi K2.5 vision
|
||||
# preprocessing via kimi_k25_vision_fused). A separate warmup Thread
|
||||
# would run concurrently with that single serving worker, so both
|
||||
# threads would enter the numba workqueue parallel region at the same
|
||||
# time and trip numba's "Concurrent access has been detected" ->
|
||||
# "Fatal Python error: Aborted" (see vllm/utils/jit_monitor.py, which
|
||||
# forces the non-threadsafe workqueue layer). Submitting the warmup to
|
||||
# the *same* single-worker executor makes warmup and serving
|
||||
# physically incapable of overlapping on the numba threading layer,
|
||||
# while still overlapping at the process level with the (forked)
|
||||
# engine-core init in the main thread.
|
||||
#
|
||||
# The warmup task is joined by the first of: reset_mm_cache, warmup, or
|
||||
# shutdown. This also guarantees the frontend mm_processor_cache —
|
||||
# which is not designed for concurrent access (its get_and_update must
|
||||
# stay ordered with the engine-core cache) — is never touched
|
||||
# concurrently by the warmup and the serving path.
|
||||
if self.mm_processor is None and self._readonly_mm_processor is None:
|
||||
return # text-only model: nothing to warm up
|
||||
if self._mm_warmup_future is not None:
|
||||
return # already launched
|
||||
self._mm_warmup_future = self._mm_executor.submit(self.warmup_mm)
|
||||
|
||||
def _join_mm_warmup(self) -> None:
|
||||
# Wait for the background MM warmup task to finish, if one was started.
|
||||
# Called from the main / event-loop thread (reset_mm_cache, warmup,
|
||||
# shutdown) — never from inside the ``_mm_executor`` worker — so
|
||||
# waiting on the future cannot deadlock.
|
||||
future = getattr(self, "_mm_warmup_future", None)
|
||||
if future is not None:
|
||||
future.result()
|
||||
self._mm_warmup_future = None
|
||||
|
||||
def warmup(self, chat_params: ChatParams) -> None:
|
||||
"""
|
||||
Warm up this renderer to avoid first-request latency.
|
||||
|
||||
For chat requests:
|
||||
- Jinja2 template compilation
|
||||
|
||||
If a background multimodal warmup was launched (to overlap
|
||||
engine-core init) it is joined first so it is not still touching the
|
||||
mm_processor_cache. warmup_mm is idempotent, so the call below is a
|
||||
no-op if the background warmup — or a prior reset_mm_cache that joined
|
||||
it — already ran it.
|
||||
"""
|
||||
from vllm.entrypoints.chat_utils import ChatTemplateResolutionError
|
||||
|
||||
@@ -363,12 +258,29 @@ class BaseRenderer(ABC, Generic[_T]):
|
||||
except Exception:
|
||||
logger.warning("Chat template warmup failed", exc_info=True)
|
||||
|
||||
if self._mm_warmup_future is not None:
|
||||
# Join a background MM warmup launched to overlap engine-core init.
|
||||
self._join_mm_warmup()
|
||||
# Idempotent: no-op if the background warmup (or a prior reset_mm_cache)
|
||||
# already ran the MM warmup.
|
||||
self.warmup_mm()
|
||||
if self.mm_processor:
|
||||
try:
|
||||
logger.debug("Warming up multi-modal processing...")
|
||||
self._warmup_mm_processor(
|
||||
self.mm_processor,
|
||||
log_prefix="Multi-modal",
|
||||
)
|
||||
except Exception:
|
||||
logger.warning("Multi-modal warmup failed")
|
||||
finally:
|
||||
self.clear_mm_cache()
|
||||
|
||||
if self._readonly_mm_processor is not None:
|
||||
try:
|
||||
logger.debug("Warming up readonly multi-modal processing...")
|
||||
self._warmup_mm_processor(
|
||||
self._readonly_mm_processor,
|
||||
log_prefix="Readonly multi-modal",
|
||||
)
|
||||
except Exception:
|
||||
logger.warning("Readonly multi-modal warmup failed")
|
||||
finally:
|
||||
self._clear_processor_cache(self._readonly_mm_processor)
|
||||
|
||||
async def clear_mm_cache_async(self) -> None:
|
||||
"""Serialize clear_mm_cache through the multimodal executor to avoid
|
||||
@@ -376,10 +288,6 @@ class BaseRenderer(ABC, Generic[_T]):
|
||||
await self._clear_mm_cache_async()
|
||||
|
||||
def shutdown(self) -> None:
|
||||
# Wait for any background MM warmup to finish before closing the
|
||||
# mm_processor_cache and stopping the executors.
|
||||
self._join_mm_warmup()
|
||||
|
||||
mm_processor_cache = self.mm_processor_cache
|
||||
if mm_processor_cache is not None:
|
||||
mm_processor_cache.close()
|
||||
|
||||
@@ -137,19 +137,6 @@ class AsyncLLM(EngineClient):
|
||||
# Convert EngineInput --> EngineCoreRequest.
|
||||
self.input_processor = InputProcessor(self.vllm_config, renderer)
|
||||
|
||||
# Launch the multimodal warmup in the background so it overlaps
|
||||
# engine-core initialization (model loading, started below). Must be
|
||||
# after InputProcessor: its MultiModalBudget runs a synchronous MM
|
||||
# processing pass (get_dummy_mm_inputs -> processor.apply), and the HF
|
||||
# processor's Numba parallel kernels (e.g. Kimi-K2.5 vision fused) are
|
||||
# not thread-safe — launching the warmup thread any earlier would run
|
||||
# it concurrently with that pass and abort. The MM warmup is
|
||||
# frontend-only (sends no engine_core requests) and only needs the
|
||||
# renderer. Joined by reset_mm_cache / warmup / shutdown so the
|
||||
# mm_processor_cache is never touched concurrently by the warmup and
|
||||
# the serving path.
|
||||
renderer.start_mm_warmup_in_background()
|
||||
|
||||
# Converts EngineCoreOutputs --> RequestOutput.
|
||||
self.output_processor = OutputProcessor(
|
||||
renderer.tokenizer,
|
||||
@@ -969,10 +956,6 @@ class AsyncLLM(EngineClient):
|
||||
await asyncio.gather(*coros)
|
||||
|
||||
async def reset_mm_cache(self) -> None:
|
||||
# Join any background MM warmup before clearing: the mm_processor_cache
|
||||
# is not designed for concurrent access, so the warmup's apply/clear
|
||||
# must have finished before we clear here.
|
||||
self.renderer._join_mm_warmup()
|
||||
await self.renderer.clear_mm_cache_async()
|
||||
await self.engine_core.reset_mm_cache_async()
|
||||
|
||||
|
||||
@@ -93,19 +93,6 @@ class LLMEngine:
|
||||
# Convert EngineInput --> EngineCoreRequest.
|
||||
self.input_processor = InputProcessor(self.vllm_config, renderer)
|
||||
|
||||
# Launch the multimodal warmup in the background so it overlaps
|
||||
# engine-core initialization (model loading, started below). Must be
|
||||
# after InputProcessor: its MultiModalBudget runs a synchronous MM
|
||||
# processing pass (get_dummy_mm_inputs -> processor.apply), and the HF
|
||||
# processor's Numba parallel kernels (e.g. Kimi-K2.5 vision fused) are
|
||||
# not thread-safe — launching the warmup thread any earlier would run
|
||||
# it concurrently with that pass and abort. The MM warmup is
|
||||
# frontend-only (sends no engine_core requests) and only needs the
|
||||
# renderer. Joined by reset_mm_cache / warmup / shutdown so the
|
||||
# mm_processor_cache is never touched concurrently by the warmup and
|
||||
# the serving path.
|
||||
renderer.start_mm_warmup_in_background()
|
||||
|
||||
# Converts EngineCoreOutputs --> RequestOutput.
|
||||
self.output_processor = OutputProcessor(
|
||||
renderer.tokenizer,
|
||||
@@ -356,10 +343,6 @@ class LLMEngine:
|
||||
self.engine_core.profile(False)
|
||||
|
||||
def reset_mm_cache(self):
|
||||
# Join any background MM warmup before clearing: the mm_processor_cache
|
||||
# is not designed for concurrent access, so the warmup's apply/clear
|
||||
# must have finished before we clear here.
|
||||
self.renderer._join_mm_warmup()
|
||||
self.renderer.clear_mm_cache()
|
||||
self.engine_core.reset_mm_cache()
|
||||
|
||||
|
||||
Reference in New Issue
Block a user