diff --git a/tests/renderers/test_warmup.py b/tests/renderers/test_warmup.py index e3f1a1607e..ac951aae46 100644 --- a/tests/renderers/test_warmup.py +++ b/tests/renderers/test_warmup.py @@ -6,18 +6,11 @@ These tests exercise: - Zero-limit modalities are filtered from mm_counts passed to get_dummy_processor_inputs (e.g. --limit-mm-per-prompt image=0 ...) - MM warmup is skipped entirely when mm_processor is None - - The multimodal warmup is launched as a task on the single-worker - ``_mm_executor`` to overlap engine-core init (future lifecycle, join by - warmup/reset/shutdown, no double-run). Routing it through the same - executor that serves ``_process_multimodal`` keeps the numba workqueue - parallel region single-threaded, avoiding the "Concurrent access has been - detected" fatal abort when a request arrives during warmup. No model weights are required: warmup() is called directly on a MagicMock that acts as the renderer instance. """ -from concurrent.futures import Future, ThreadPoolExecutor from unittest.mock import MagicMock, patch from vllm.renderers.base import BaseRenderer @@ -46,18 +39,6 @@ def _make_renderer_mock(mm_limits: dict[str, int]) -> MagicMock: renderer, BaseRenderer ) renderer._clear_processor_cache = BaseRenderer._clear_processor_cache - renderer.warmup_mm = BaseRenderer.warmup_mm.__get__(renderer, BaseRenderer) - renderer.start_mm_warmup_in_background = ( - BaseRenderer.start_mm_warmup_in_background.__get__(renderer, BaseRenderer) - ) - renderer._join_mm_warmup = BaseRenderer._join_mm_warmup.__get__( - renderer, BaseRenderer - ) - renderer.shutdown = BaseRenderer.shutdown.__get__(renderer, BaseRenderer) - # No background warmup launched by default; warmup() takes the inline path. - renderer._mm_warmup_future = None - # MM warmup has not run yet; warmup_mm must actually execute on the mock. - renderer._mm_warmup_done = False renderer.clear_mm_cache = MagicMock() renderer.model_config.max_model_len = 128 renderer.model_config.get_multimodal_config.return_value.limit_per_prompt = {} @@ -152,179 +133,3 @@ class TestReadonlyMmWarmup: readonly_mm_processor.apply.assert_called_once() readonly_mm_processor.cache.clear_cache.assert_called_once() - - -class TestChatWarmupDispatched: - """The chat-template warmup task must always run.""" - - def test_render_chat_called_with_warmup_message(self): - renderer = _make_renderer_mock({"image": 1}) - - with patch("vllm.multimodal.processing.TimingContext", autospec=True): - BaseRenderer.warmup(renderer, ChatParams()) - - renderer.render_chat.assert_called_once() - - -class TestWarmupFaultIsolation: - """A failure during a multimodal processor warmup is caught so it does not - abort the remaining warmup steps; warmup itself must not raise.""" - - def test_chat_failure_does_not_abort_mm_warmup(self): - renderer = _make_renderer_mock({"image": 1}) - renderer.render_chat.side_effect = RuntimeError("chat boom") - - with patch("vllm.multimodal.processing.TimingContext", autospec=True): - BaseRenderer.warmup(renderer, ChatParams()) # must not raise - - renderer.mm_processor.apply.assert_called_once() - - def test_mm_failure_does_not_abort_readonly_warmup(self): - renderer = _make_renderer_mock({"image": 1}) - readonly_mm_processor = MagicMock() - readonly_mm_processor.info.allowed_mm_limits = {"image": 1} - renderer._readonly_mm_processor = readonly_mm_processor - # main processor warmup blows up before apply() - renderer.mm_processor.dummy_inputs.get_dummy_processor_inputs.side_effect = ( - RuntimeError("mm boom") - ) - - with patch("vllm.multimodal.processing.TimingContext", autospec=True): - BaseRenderer.warmup(renderer, ChatParams()) # must not raise - - readonly_mm_processor.apply.assert_called_once() - # cache is still cleared in the failed task's finally - renderer.clear_mm_cache.assert_called_once() - - -class TestBackgroundMmWarmup: - """The multimodal warmup is launched as a task on the single-worker - ``_mm_executor`` so it overlaps engine-core init (fork) while staying - serialized with the serving path (``_process_multimodal`` runs on the same - executor). warmup()/reset_mm_cache/shutdown join it. This also fixes the - numba workqueue "Concurrent access has been detected" abort: warmup and a - concurrent serving request can no longer both enter the numba parallel - region at once.""" - - def _make_bg_renderer(self, mm_limits: dict[str, int]): - """A renderer mock with a real single-worker executor so submit() - returns a real Future and warmup_mm actually runs (in the worker).""" - renderer = _make_renderer_mock(mm_limits) - renderer._mm_executor = ThreadPoolExecutor(max_workers=1) - return renderer - - def test_start_submits_future_to_mm_executor(self): - renderer = self._make_bg_renderer({"image": 1}) - try: - with patch("vllm.multimodal.processing.TimingContext", autospec=True): - # Spy on the real executor's submit to confirm the warmup is - # dispatched through _mm_executor (not a separate Thread). - with patch.object( - renderer._mm_executor, "submit", wraps=renderer._mm_executor.submit - ) as spy: - renderer.start_mm_warmup_in_background() - assert isinstance(renderer._mm_warmup_future, Future) - assert spy.called - renderer._join_mm_warmup() - assert renderer._mm_warmup_future is None - renderer.mm_processor.apply.assert_called_once() - finally: - renderer._mm_executor.shutdown(wait=True) - - def test_start_noop_for_text_only_model(self): - renderer = _make_renderer_mock({}) - renderer.mm_processor = None - # _readonly_mm_processor is already None - - renderer.start_mm_warmup_in_background() - - assert renderer._mm_warmup_future is None - - def test_start_is_run_at_most_once(self): - renderer = self._make_bg_renderer({"image": 1}) - try: - with patch("vllm.multimodal.processing.TimingContext", autospec=True): - renderer.start_mm_warmup_in_background() - first = renderer._mm_warmup_future - renderer.start_mm_warmup_in_background() # must not spawn a second - assert renderer._mm_warmup_future is first - renderer._join_mm_warmup() - finally: - renderer._mm_executor.shutdown(wait=True) - - def test_warmup_joins_background_and_does_not_rerun_mm(self): - """When a background MM warmup is in flight, warmup() must join it and - run only the chat warmup — the MM warmup must not run twice.""" - renderer = self._make_bg_renderer({"image": 1}) - try: - with patch("vllm.multimodal.processing.TimingContext", autospec=True): - renderer.start_mm_warmup_in_background() - BaseRenderer.warmup(renderer, ChatParams()) - - # MM apply called exactly once (by the background warmup task). - renderer.mm_processor.apply.assert_called_once() - # Chat warmup ran exactly once. - renderer.render_chat.assert_called_once() - # Background task has been joined and cleared. - assert renderer._mm_warmup_future is None - finally: - renderer._mm_executor.shutdown(wait=True) - - def test_warmup_does_not_rerun_mm_after_reset_joins_background(self): - """Regression: reset_mm_cache joins the background warmup before - warmup() runs (it clears _mm_warmup_future). warmup() must still not - re-run the MM warmup — the _mm_warmup_done flag survives the join.""" - renderer = self._make_bg_renderer({"image": 1}) - try: - with patch("vllm.multimodal.processing.TimingContext", autospec=True): - renderer.start_mm_warmup_in_background() - # Simulate reset_mm_cache joining the background warmup first - # (clears _mm_warmup_future, sets _mm_warmup_done via the join). - renderer._join_mm_warmup() - assert renderer._mm_warmup_future is None # joined & cleared - BaseRenderer.warmup(renderer, ChatParams()) - - # MM apply called exactly once (by the background warmup task); - # warmup() did not re-run warmup_mm despite the future being cleared. - renderer.mm_processor.apply.assert_called_once() - renderer.render_chat.assert_called_once() - finally: - renderer._mm_executor.shutdown(wait=True) - - def test_warmup_mm_runs_at_most_once(self): - """Direct repeated calls to warmup_mm run the MM warmup only once.""" - renderer = _make_renderer_mock({"image": 1}) - - with patch("vllm.multimodal.processing.TimingContext", autospec=True): - renderer.warmup_mm() - renderer.warmup_mm() # second call must be a no-op - - renderer.mm_processor.apply.assert_called_once() - - def test_join_mm_warmup_blocks_until_apply_done(self): - renderer = self._make_bg_renderer({"image": 1}) - try: - with patch("vllm.multimodal.processing.TimingContext", autospec=True): - renderer.start_mm_warmup_in_background() - renderer._join_mm_warmup() - - assert renderer._mm_warmup_future is None - renderer.mm_processor.apply.assert_called_once() - finally: - renderer._mm_executor.shutdown(wait=True) - - def test_shutdown_joins_background_warmup(self): - """shutdown() must join the background warmup before closing caches, - so the mm_processor_cache is never touched concurrently.""" - renderer = self._make_bg_renderer({"image": 1}) - - with patch("vllm.multimodal.processing.TimingContext", autospec=True): - renderer.start_mm_warmup_in_background() - future = renderer._mm_warmup_future - renderer.shutdown() - - assert renderer._mm_warmup_future is None - # The background warmup was allowed to complete (apply ran) and the - # future is done after shutdown joined it. - assert future.done() - renderer.mm_processor.apply.assert_called_once() diff --git a/vllm/renderers/base.py b/vllm/renderers/base.py index 627175bb08..aaf89e6edd 100644 --- a/vllm/renderers/base.py +++ b/vllm/renderers/base.py @@ -4,7 +4,7 @@ import asyncio import time from abc import ABC, abstractmethod from collections.abc import Mapping, Sequence -from concurrent.futures import Executor, Future, ThreadPoolExecutor +from concurrent.futures import Executor, ThreadPoolExecutor from functools import cached_property from typing import TYPE_CHECKING, Any, Generic, overload @@ -79,26 +79,6 @@ class BaseRenderer(ABC, Generic[_T]): self.tokenizer = tokenizer - # Background multimodal warmup task (a future returned by - # ``_mm_executor``) so it overlaps engine-core initialization while - # staying serialized with the serving path through the same - # single-worker executor. None when no background warmup has been - # launched (or it has already been joined). Routing the warmup through - # ``_mm_executor`` (max_workers=1) guarantees it can never run - # concurrently with a serving request inside the numba workqueue - # parallel region, which would otherwise hit numba's - # "Concurrent access has been detected" fatal abort (see - # vllm/utils/jit_monitor.py, which forces the workqueue layer). - # Joined by the first of reset_mm_cache / warmup / shutdown so the - # mm_processor_cache is also never touched concurrently by the warmup - # and the serving path. - self._mm_warmup_future: Future[None] | None = None - # Whether warmup_mm has already run. Survives _join_mm_warmup (which - # clears the thread pointer), so warmup() can tell that the background - # warmup already ran even after reset_mm_cache joined it first, and - # avoid running warmup_mm a second time. - self._mm_warmup_done: bool = False - # Thread pool executor for blocking tokenizer operations. The # multimodal processor receives a deep-copied tokenizer (see #36557) # so it is safe to run tokenization and MM preprocessing concurrently. @@ -254,97 +234,12 @@ class BaseRenderer(ABC, Generic[_T]): elapsed = time.perf_counter() - start_time logger.info("%s warmup completed in %.3fs", log_prefix, elapsed) - def warmup_mm(self) -> None: - # Run only the multimodal warmup (mm + readonly mm), serially. - # Idempotent: runs at most once. A background warmup - # (start_mm_warmup_in_background) may already have run it, and - # reset_mm_cache may have joined that background warmup task before - # warmup() reaches this point — the _mm_warmup_done flag survives the - # join (the future pointer does not), so warmup() can still tell it - # already ran. - if self._mm_warmup_done: - return - try: - # prevent MM processor hangs - with set_default_torch_num_threads(1): - if self.mm_processor: - try: - logger.debug("Warming up multi-modal processing...") - self._warmup_mm_processor( - self.mm_processor, - log_prefix="Multi-modal", - ) - except Exception: - logger.warning("Multi-modal warmup failed") - finally: - self.clear_mm_cache() - - if self._readonly_mm_processor is not None: - try: - logger.debug("Warming up readonly multi-modal processing...") - self._warmup_mm_processor( - self._readonly_mm_processor, - log_prefix="Readonly multi-modal", - ) - except Exception: - logger.warning("Readonly multi-modal warmup failed") - finally: - self._clear_processor_cache(self._readonly_mm_processor) - finally: - self._mm_warmup_done = True - - def start_mm_warmup_in_background(self) -> None: - # Launch the multimodal warmup as a task on the single-worker - # ``_mm_executor`` so it overlaps with engine-core initialization - # (model loading) while staying serialized with the serving path. - # - # Why the *same* executor as serving (not a separate Thread): - # ``_process_multimodal`` runs on ``_mm_executor`` (max_workers=1) and - # enters the numba workqueue parallel region (e.g. Kimi K2.5 vision - # preprocessing via kimi_k25_vision_fused). A separate warmup Thread - # would run concurrently with that single serving worker, so both - # threads would enter the numba workqueue parallel region at the same - # time and trip numba's "Concurrent access has been detected" -> - # "Fatal Python error: Aborted" (see vllm/utils/jit_monitor.py, which - # forces the non-threadsafe workqueue layer). Submitting the warmup to - # the *same* single-worker executor makes warmup and serving - # physically incapable of overlapping on the numba threading layer, - # while still overlapping at the process level with the (forked) - # engine-core init in the main thread. - # - # The warmup task is joined by the first of: reset_mm_cache, warmup, or - # shutdown. This also guarantees the frontend mm_processor_cache — - # which is not designed for concurrent access (its get_and_update must - # stay ordered with the engine-core cache) — is never touched - # concurrently by the warmup and the serving path. - if self.mm_processor is None and self._readonly_mm_processor is None: - return # text-only model: nothing to warm up - if self._mm_warmup_future is not None: - return # already launched - self._mm_warmup_future = self._mm_executor.submit(self.warmup_mm) - - def _join_mm_warmup(self) -> None: - # Wait for the background MM warmup task to finish, if one was started. - # Called from the main / event-loop thread (reset_mm_cache, warmup, - # shutdown) — never from inside the ``_mm_executor`` worker — so - # waiting on the future cannot deadlock. - future = getattr(self, "_mm_warmup_future", None) - if future is not None: - future.result() - self._mm_warmup_future = None - def warmup(self, chat_params: ChatParams) -> None: """ Warm up this renderer to avoid first-request latency. For chat requests: - Jinja2 template compilation - - If a background multimodal warmup was launched (to overlap - engine-core init) it is joined first so it is not still touching the - mm_processor_cache. warmup_mm is idempotent, so the call below is a - no-op if the background warmup — or a prior reset_mm_cache that joined - it — already ran it. """ from vllm.entrypoints.chat_utils import ChatTemplateResolutionError @@ -363,12 +258,29 @@ class BaseRenderer(ABC, Generic[_T]): except Exception: logger.warning("Chat template warmup failed", exc_info=True) - if self._mm_warmup_future is not None: - # Join a background MM warmup launched to overlap engine-core init. - self._join_mm_warmup() - # Idempotent: no-op if the background warmup (or a prior reset_mm_cache) - # already ran the MM warmup. - self.warmup_mm() + if self.mm_processor: + try: + logger.debug("Warming up multi-modal processing...") + self._warmup_mm_processor( + self.mm_processor, + log_prefix="Multi-modal", + ) + except Exception: + logger.warning("Multi-modal warmup failed") + finally: + self.clear_mm_cache() + + if self._readonly_mm_processor is not None: + try: + logger.debug("Warming up readonly multi-modal processing...") + self._warmup_mm_processor( + self._readonly_mm_processor, + log_prefix="Readonly multi-modal", + ) + except Exception: + logger.warning("Readonly multi-modal warmup failed") + finally: + self._clear_processor_cache(self._readonly_mm_processor) async def clear_mm_cache_async(self) -> None: """Serialize clear_mm_cache through the multimodal executor to avoid @@ -376,10 +288,6 @@ class BaseRenderer(ABC, Generic[_T]): await self._clear_mm_cache_async() def shutdown(self) -> None: - # Wait for any background MM warmup to finish before closing the - # mm_processor_cache and stopping the executors. - self._join_mm_warmup() - mm_processor_cache = self.mm_processor_cache if mm_processor_cache is not None: mm_processor_cache.close() diff --git a/vllm/v1/engine/async_llm.py b/vllm/v1/engine/async_llm.py index 417b37b610..dd5cb2c262 100644 --- a/vllm/v1/engine/async_llm.py +++ b/vllm/v1/engine/async_llm.py @@ -137,19 +137,6 @@ class AsyncLLM(EngineClient): # Convert EngineInput --> EngineCoreRequest. self.input_processor = InputProcessor(self.vllm_config, renderer) - # Launch the multimodal warmup in the background so it overlaps - # engine-core initialization (model loading, started below). Must be - # after InputProcessor: its MultiModalBudget runs a synchronous MM - # processing pass (get_dummy_mm_inputs -> processor.apply), and the HF - # processor's Numba parallel kernels (e.g. Kimi-K2.5 vision fused) are - # not thread-safe — launching the warmup thread any earlier would run - # it concurrently with that pass and abort. The MM warmup is - # frontend-only (sends no engine_core requests) and only needs the - # renderer. Joined by reset_mm_cache / warmup / shutdown so the - # mm_processor_cache is never touched concurrently by the warmup and - # the serving path. - renderer.start_mm_warmup_in_background() - # Converts EngineCoreOutputs --> RequestOutput. self.output_processor = OutputProcessor( renderer.tokenizer, @@ -969,10 +956,6 @@ class AsyncLLM(EngineClient): await asyncio.gather(*coros) async def reset_mm_cache(self) -> None: - # Join any background MM warmup before clearing: the mm_processor_cache - # is not designed for concurrent access, so the warmup's apply/clear - # must have finished before we clear here. - self.renderer._join_mm_warmup() await self.renderer.clear_mm_cache_async() await self.engine_core.reset_mm_cache_async() diff --git a/vllm/v1/engine/llm_engine.py b/vllm/v1/engine/llm_engine.py index d47035e8ed..0db83b95f2 100644 --- a/vllm/v1/engine/llm_engine.py +++ b/vllm/v1/engine/llm_engine.py @@ -93,19 +93,6 @@ class LLMEngine: # Convert EngineInput --> EngineCoreRequest. self.input_processor = InputProcessor(self.vllm_config, renderer) - # Launch the multimodal warmup in the background so it overlaps - # engine-core initialization (model loading, started below). Must be - # after InputProcessor: its MultiModalBudget runs a synchronous MM - # processing pass (get_dummy_mm_inputs -> processor.apply), and the HF - # processor's Numba parallel kernels (e.g. Kimi-K2.5 vision fused) are - # not thread-safe — launching the warmup thread any earlier would run - # it concurrently with that pass and abort. The MM warmup is - # frontend-only (sends no engine_core requests) and only needs the - # renderer. Joined by reset_mm_cache / warmup / shutdown so the - # mm_processor_cache is never touched concurrently by the warmup and - # the serving path. - renderer.start_mm_warmup_in_background() - # Converts EngineCoreOutputs --> RequestOutput. self.output_processor = OutputProcessor( renderer.tokenizer, @@ -356,10 +343,6 @@ class LLMEngine: self.engine_core.profile(False) def reset_mm_cache(self): - # Join any background MM warmup before clearing: the mm_processor_cache - # is not designed for concurrent access, so the warmup's apply/clear - # must have finished before we clear here. - self.renderer._join_mm_warmup() self.renderer.clear_mm_cache() self.engine_core.reset_mm_cache()