From f5ffc59b6abbf317708d4c319702bac2b2aa8e72 Mon Sep 17 00:00:00 2001 From: "wang.yuqi" Date: Fri, 31 Jul 2026 19:46:53 +0800 Subject: [PATCH] [Renderer] Warm up the renderer properly. (#50408) Signed-off-by: wang.yuqi --- vllm/entrypoints/generate/api_router.py | 2 - vllm/entrypoints/llm.py | 3 + vllm/entrypoints/openai/api_server.py | 2 + .../openai/chat_completion/serving.py | 10 --- vllm/renderers/base.py | 68 ++++++++++--------- vllm/renderers/online_renderer.py | 11 ++- 6 files changed, 50 insertions(+), 46 deletions(-) diff --git a/vllm/entrypoints/generate/api_router.py b/vllm/entrypoints/generate/api_router.py index 062a7947e7..adbcc4cd3f 100644 --- a/vllm/entrypoints/generate/api_router.py +++ b/vllm/entrypoints/generate/api_router.py @@ -140,8 +140,6 @@ async def init_generate_state( if "generate" in supported_tasks else None ) - if state.openai_serving_chat is not None: - state.openai_serving_chat.warmup() state.openai_serving_completion = ( OpenAIServingCompletion( engine_client, diff --git a/vllm/entrypoints/llm.py b/vllm/entrypoints/llm.py index 4274819d98..478aa80357 100644 --- a/vllm/entrypoints/llm.py +++ b/vllm/entrypoints/llm.py @@ -55,6 +55,7 @@ from vllm.v1.engine import PauseMode from vllm.v1.engine.llm_engine import LLMEngine from vllm.v1.sample.logits_processor import LogitsProcessor +from ..renderers import ChatParams from .offline_utils import _O, _R, OfflineInferenceMixin if TYPE_CHECKING: @@ -352,6 +353,8 @@ class LLM(BeamSearchOfflineMixin, PoolingOfflineMixin, OfflineInferenceMixin): self.chat_template = load_chat_template(chat_template) self.input_processor = self.llm_engine.input_processor + self.renderer.warmup(ChatParams(chat_template=self.chat_template)) + # The renderer thread pool is only consumed by the async renderer # path; the synchronous `LLM` entrypoint runs multimodal # preprocessing serially. Warn so the setting is not a silent diff --git a/vllm/entrypoints/openai/api_server.py b/vllm/entrypoints/openai/api_server.py index f57e320a90..5a6dd8c83d 100644 --- a/vllm/entrypoints/openai/api_server.py +++ b/vllm/entrypoints/openai/api_server.py @@ -426,6 +426,7 @@ async def init_app_state( default_chat_template_kwargs=args.default_chat_template_kwargs, log_error_stack=args.log_error_stack, ) + state.online_renderer.warmup() state.online_derenderer = OnlineDerenderer( model_config=engine_client.model_config, @@ -529,6 +530,7 @@ async def init_render_app_state( default_chat_template_kwargs=args.default_chat_template_kwargs, log_error_stack=args.log_error_stack, ) + state.online_renderer.warmup() state.online_derenderer = OnlineDerenderer( model_config=vllm_config.model_config, diff --git a/vllm/entrypoints/openai/chat_completion/serving.py b/vllm/entrypoints/openai/chat_completion/serving.py index f149a305f6..e78e6d58e0 100644 --- a/vllm/entrypoints/openai/chat_completion/serving.py +++ b/vllm/entrypoints/openai/chat_completion/serving.py @@ -60,7 +60,6 @@ from vllm.logprobs import Logprob from vllm.outputs import RequestOutput from vllm.parser import ParserManager from vllm.parser.abstract_parser import Parser -from vllm.renderers import ChatParams from vllm.renderers.online_renderer import OnlineRenderer from vllm.sampling_params import BeamSearchParams, SamplingParams from vllm.tokenizers import TokenizerLike @@ -178,15 +177,6 @@ class OpenAIServingChat(GenerateBaseServing): self.supports_code_interpreter = False self.python_tool = None - def warmup(self) -> None: - self.renderer.warmup( - ChatParams( - chat_template=self.chat_template, - chat_template_content_format=self.chat_template_content_format, - chat_template_kwargs=self.default_chat_template_kwargs, - ) - ) - def _effective_chat_template_kwargs( self, request: ChatCompletionRequest ) -> dict[str, Any]: diff --git a/vllm/renderers/base.py b/vllm/renderers/base.py index b278a8c13d..832f954eaf 100644 --- a/vllm/renderers/base.py +++ b/vllm/renderers/base.py @@ -243,42 +243,44 @@ class BaseRenderer(ABC, Generic[_T]): """ from vllm.entrypoints.chat_utils import ChatTemplateResolutionError - try: - logger.debug("Warming up chat template processing...") - start_time = time.perf_counter() - - self.render_chat([[{"role": "user", "content": "warmup"}]], chat_params) - - elapsed = time.perf_counter() - start_time - logger.debug("Chat template warmup completed in %.3fs", elapsed) - except ChatTemplateResolutionError: - logger.debug("This model does not support chat template.") - except Exception: - logger.warning("Chat template warmup failed", exc_info=True) - - if self.mm_processor: + # prevent MM processor hangs + with set_default_torch_num_threads(1): try: - logger.debug("Warming up multi-modal processing...") - self._warmup_mm_processor( - self.mm_processor, - log_prefix="Multi-modal", - ) - except Exception: - logger.warning("Multi-modal warmup failed") - finally: - self.clear_mm_cache() + logger.debug("Warming up chat template processing...") + start_time = time.perf_counter() - if self._readonly_mm_processor is not None: - try: - logger.debug("Warming up readonly multi-modal processing...") - self._warmup_mm_processor( - self._readonly_mm_processor, - log_prefix="Readonly multi-modal", - ) + self.render_chat([[{"role": "user", "content": "warmup"}]], chat_params) + + elapsed = time.perf_counter() - start_time + logger.debug("Chat template warmup completed in %.3fs", elapsed) + except ChatTemplateResolutionError: + logger.debug("This model does not support chat template.") except Exception: - logger.warning("Readonly multi-modal warmup failed") - finally: - self._clear_processor_cache(self._readonly_mm_processor) + logger.warning("Chat template warmup failed", exc_info=True) + + if self.mm_processor: + try: + logger.debug("Warming up multi-modal processing...") + self._warmup_mm_processor( + self.mm_processor, + log_prefix="Multi-modal", + ) + except Exception: + logger.warning("Multi-modal warmup failed") + finally: + self.clear_mm_cache() + + if self._readonly_mm_processor is not None: + try: + logger.debug("Warming up readonly multi-modal processing...") + self._warmup_mm_processor( + self._readonly_mm_processor, + log_prefix="Readonly multi-modal", + ) + except Exception: + logger.warning("Readonly multi-modal warmup failed") + finally: + self._clear_processor_cache(self._readonly_mm_processor) async def clear_mm_cache_async(self) -> None: """Serialize clear_mm_cache through the multimodal executor to avoid diff --git a/vllm/renderers/online_renderer.py b/vllm/renderers/online_renderer.py index 6303d0d942..995527654f 100644 --- a/vllm/renderers/online_renderer.py +++ b/vllm/renderers/online_renderer.py @@ -36,7 +36,7 @@ from vllm.inputs import ( ) from vllm.logger import init_logger from vllm.parser import Parser, ParserManager -from vllm.renderers import BaseRenderer, merge_kwargs +from vllm.renderers import BaseRenderer, ChatParams, merge_kwargs from vllm.renderers.inputs.preprocess import ( parse_model_prompt, prompt_to_seq, @@ -105,6 +105,15 @@ class OnlineRenderer: self.supports_browsing = False self.supports_code_interpreter = False + def warmup(self) -> None: + self.renderer.warmup( + ChatParams( + chat_template=self.chat_template, + chat_template_content_format=self.chat_template_content_format, + chat_template_kwargs=self.default_chat_template_kwargs, + ) + ) + async def render_chat( self, request: ChatCompletionRequest,