[Feature] Warm up readonly multimodal processor during renderer startup (#40797)
Signed-off-by: Chenguang ZHENG <645327136@qq.com> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
co-authored by
OpenAI Codex
parent
428b988c98
commit
07351e0883
@@ -136,7 +136,7 @@ class OpenAIServingRender:
|
||||
"Beam search is not supported by the render endpoint"
|
||||
)
|
||||
|
||||
result = await self.render_chat(request)
|
||||
result = await self.render_chat(request, skip_mm_cache=True)
|
||||
if isinstance(result, ErrorResponse):
|
||||
return result
|
||||
|
||||
@@ -184,6 +184,8 @@ class OpenAIServingRender:
|
||||
async def render_chat(
|
||||
self,
|
||||
request: ChatCompletionRequest,
|
||||
*,
|
||||
skip_mm_cache: bool = False,
|
||||
) -> tuple[list[ConversationMessage], list[EngineInput]] | ErrorResponse:
|
||||
"""Core preprocessing logic for chat requests (no model/engine check).
|
||||
|
||||
@@ -252,7 +254,7 @@ class OpenAIServingRender:
|
||||
default_template_kwargs=self.default_chat_template_kwargs,
|
||||
tool_dicts=tool_dicts,
|
||||
tool_parser=tool_parser,
|
||||
skip_mm_cache=True,
|
||||
skip_mm_cache=skip_mm_cache,
|
||||
reasoning_parser=self.reasoning_parser,
|
||||
)
|
||||
else:
|
||||
@@ -276,7 +278,7 @@ class OpenAIServingRender:
|
||||
error_check_ret = await self._check_model(request)
|
||||
if error_check_ret is not None:
|
||||
return error_check_ret
|
||||
result = await self.render_completion(request)
|
||||
result = await self.render_completion(request, skip_mm_cache=True)
|
||||
if isinstance(result, ErrorResponse):
|
||||
return result
|
||||
generate_requests: list[GenerateRequest] = []
|
||||
@@ -322,6 +324,8 @@ class OpenAIServingRender:
|
||||
async def render_completion(
|
||||
self,
|
||||
request: CompletionRequest,
|
||||
*,
|
||||
skip_mm_cache: bool = False,
|
||||
) -> list[EngineInput] | ErrorResponse:
|
||||
"""Core preprocessing logic for completion requests (no model/engine check).
|
||||
|
||||
@@ -344,7 +348,7 @@ class OpenAIServingRender:
|
||||
request,
|
||||
prompt_input=request.prompt,
|
||||
prompt_embeds=request.prompt_embeds,
|
||||
skip_mm_cache=True,
|
||||
skip_mm_cache=skip_mm_cache,
|
||||
)
|
||||
|
||||
return engine_inputs
|
||||
|
||||
+49
-21
@@ -198,6 +198,40 @@ class BaseRenderer(ABC, Generic[_T]):
|
||||
if self._mm_cache_stats is not None:
|
||||
self._mm_cache_stats.reset = True
|
||||
|
||||
@staticmethod
|
||||
def _clear_processor_cache(
|
||||
processor: "BaseMultiModalProcessor | None",
|
||||
) -> None:
|
||||
if processor is None:
|
||||
return
|
||||
|
||||
processor_cache = processor.cache
|
||||
if processor_cache is not None:
|
||||
processor_cache.clear_cache()
|
||||
|
||||
def _warmup_mm_processor(
|
||||
self,
|
||||
processor: "BaseMultiModalProcessor",
|
||||
*,
|
||||
log_prefix: str,
|
||||
) -> None:
|
||||
from vllm.multimodal.processing import TimingContext
|
||||
|
||||
model_config = self.model_config
|
||||
mm_config = model_config.get_multimodal_config()
|
||||
mm_limits = {k: v for k, v in processor.info.allowed_mm_limits.items() if v > 0}
|
||||
|
||||
start_time = time.perf_counter()
|
||||
processor_inputs = processor.dummy_inputs.get_dummy_processor_inputs(
|
||||
seq_len=model_config.max_model_len,
|
||||
mm_counts=dict.fromkeys(mm_limits, 1),
|
||||
mm_options=mm_config.limit_per_prompt,
|
||||
)
|
||||
_ = processor.apply(processor_inputs, timing_ctx=TimingContext(enabled=False))
|
||||
|
||||
elapsed = time.perf_counter() - start_time
|
||||
logger.info("%s warmup completed in %.3fs", log_prefix, elapsed)
|
||||
|
||||
def warmup(self, chat_params: ChatParams) -> None:
|
||||
"""
|
||||
Warm up this renderer to avoid first-request latency.
|
||||
@@ -221,35 +255,29 @@ class BaseRenderer(ABC, Generic[_T]):
|
||||
logger.warning("Chat template warmup failed", exc_info=True)
|
||||
|
||||
if self.mm_processor:
|
||||
from vllm.multimodal.processing import TimingContext
|
||||
|
||||
model_config = self.model_config
|
||||
mm_config = model_config.get_multimodal_config()
|
||||
processor = self.mm_processor
|
||||
mm_limits = {
|
||||
k: v for k, v in processor.info.allowed_mm_limits.items() if v > 0
|
||||
}
|
||||
|
||||
try:
|
||||
logger.debug("Warming up multi-modal processing...")
|
||||
start_time = time.perf_counter()
|
||||
|
||||
processor_inputs = processor.dummy_inputs.get_dummy_processor_inputs(
|
||||
seq_len=model_config.max_model_len,
|
||||
mm_counts=dict.fromkeys(mm_limits, 1),
|
||||
mm_options=mm_config.limit_per_prompt,
|
||||
self._warmup_mm_processor(
|
||||
self.mm_processor,
|
||||
log_prefix="Multi-modal",
|
||||
)
|
||||
_ = processor.apply(
|
||||
processor_inputs, timing_ctx=TimingContext(enabled=False)
|
||||
)
|
||||
|
||||
elapsed = time.perf_counter() - start_time
|
||||
logger.info("Multi-modal warmup completed in %.3fs", elapsed)
|
||||
except Exception:
|
||||
logger.warning("Multi-modal warmup failed")
|
||||
finally:
|
||||
self.clear_mm_cache()
|
||||
|
||||
if self._readonly_mm_processor is not None:
|
||||
try:
|
||||
logger.debug("Warming up readonly multi-modal processing...")
|
||||
self._warmup_mm_processor(
|
||||
self._readonly_mm_processor,
|
||||
log_prefix="Readonly multi-modal",
|
||||
)
|
||||
except Exception:
|
||||
logger.warning("Readonly multi-modal warmup failed")
|
||||
finally:
|
||||
self._clear_processor_cache(self._readonly_mm_processor)
|
||||
|
||||
async def clear_mm_cache_async(self) -> None:
|
||||
"""Serialize clear_mm_cache through the shared executor to avoid
|
||||
races with concurrent process_inputs on the mm_processor_cache."""
|
||||
|
||||
Reference in New Issue
Block a user