[Renderer] Warm up the renderer properly. (#50408)
Signed-off-by: wang.yuqi <yuqi.wang@daocloud.io>
This commit is contained in:
+35
-33
@@ -243,42 +243,44 @@ class BaseRenderer(ABC, Generic[_T]):
|
||||
"""
|
||||
from vllm.entrypoints.chat_utils import ChatTemplateResolutionError
|
||||
|
||||
try:
|
||||
logger.debug("Warming up chat template processing...")
|
||||
start_time = time.perf_counter()
|
||||
|
||||
self.render_chat([[{"role": "user", "content": "warmup"}]], chat_params)
|
||||
|
||||
elapsed = time.perf_counter() - start_time
|
||||
logger.debug("Chat template warmup completed in %.3fs", elapsed)
|
||||
except ChatTemplateResolutionError:
|
||||
logger.debug("This model does not support chat template.")
|
||||
except Exception:
|
||||
logger.warning("Chat template warmup failed", exc_info=True)
|
||||
|
||||
if self.mm_processor:
|
||||
# prevent MM processor hangs
|
||||
with set_default_torch_num_threads(1):
|
||||
try:
|
||||
logger.debug("Warming up multi-modal processing...")
|
||||
self._warmup_mm_processor(
|
||||
self.mm_processor,
|
||||
log_prefix="Multi-modal",
|
||||
)
|
||||
except Exception:
|
||||
logger.warning("Multi-modal warmup failed")
|
||||
finally:
|
||||
self.clear_mm_cache()
|
||||
logger.debug("Warming up chat template processing...")
|
||||
start_time = time.perf_counter()
|
||||
|
||||
if self._readonly_mm_processor is not None:
|
||||
try:
|
||||
logger.debug("Warming up readonly multi-modal processing...")
|
||||
self._warmup_mm_processor(
|
||||
self._readonly_mm_processor,
|
||||
log_prefix="Readonly multi-modal",
|
||||
)
|
||||
self.render_chat([[{"role": "user", "content": "warmup"}]], chat_params)
|
||||
|
||||
elapsed = time.perf_counter() - start_time
|
||||
logger.debug("Chat template warmup completed in %.3fs", elapsed)
|
||||
except ChatTemplateResolutionError:
|
||||
logger.debug("This model does not support chat template.")
|
||||
except Exception:
|
||||
logger.warning("Readonly multi-modal warmup failed")
|
||||
finally:
|
||||
self._clear_processor_cache(self._readonly_mm_processor)
|
||||
logger.warning("Chat template warmup failed", exc_info=True)
|
||||
|
||||
if self.mm_processor:
|
||||
try:
|
||||
logger.debug("Warming up multi-modal processing...")
|
||||
self._warmup_mm_processor(
|
||||
self.mm_processor,
|
||||
log_prefix="Multi-modal",
|
||||
)
|
||||
except Exception:
|
||||
logger.warning("Multi-modal warmup failed")
|
||||
finally:
|
||||
self.clear_mm_cache()
|
||||
|
||||
if self._readonly_mm_processor is not None:
|
||||
try:
|
||||
logger.debug("Warming up readonly multi-modal processing...")
|
||||
self._warmup_mm_processor(
|
||||
self._readonly_mm_processor,
|
||||
log_prefix="Readonly multi-modal",
|
||||
)
|
||||
except Exception:
|
||||
logger.warning("Readonly multi-modal warmup failed")
|
||||
finally:
|
||||
self._clear_processor_cache(self._readonly_mm_processor)
|
||||
|
||||
async def clear_mm_cache_async(self) -> None:
|
||||
"""Serialize clear_mm_cache through the multimodal executor to avoid
|
||||
|
||||
@@ -36,7 +36,7 @@ from vllm.inputs import (
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.parser import Parser, ParserManager
|
||||
from vllm.renderers import BaseRenderer, merge_kwargs
|
||||
from vllm.renderers import BaseRenderer, ChatParams, merge_kwargs
|
||||
from vllm.renderers.inputs.preprocess import (
|
||||
parse_model_prompt,
|
||||
prompt_to_seq,
|
||||
@@ -105,6 +105,15 @@ class OnlineRenderer:
|
||||
self.supports_browsing = False
|
||||
self.supports_code_interpreter = False
|
||||
|
||||
def warmup(self) -> None:
|
||||
self.renderer.warmup(
|
||||
ChatParams(
|
||||
chat_template=self.chat_template,
|
||||
chat_template_content_format=self.chat_template_content_format,
|
||||
chat_template_kwargs=self.default_chat_template_kwargs,
|
||||
)
|
||||
)
|
||||
|
||||
async def render_chat(
|
||||
self,
|
||||
request: ChatCompletionRequest,
|
||||
|
||||
Reference in New Issue
Block a user