[Renderer] Warm up the renderer properly. (#50408)
Signed-off-by: wang.yuqi <yuqi.wang@daocloud.io>
This commit is contained in:
@@ -140,8 +140,6 @@ async def init_generate_state(
|
|||||||
if "generate" in supported_tasks
|
if "generate" in supported_tasks
|
||||||
else None
|
else None
|
||||||
)
|
)
|
||||||
if state.openai_serving_chat is not None:
|
|
||||||
state.openai_serving_chat.warmup()
|
|
||||||
state.openai_serving_completion = (
|
state.openai_serving_completion = (
|
||||||
OpenAIServingCompletion(
|
OpenAIServingCompletion(
|
||||||
engine_client,
|
engine_client,
|
||||||
|
|||||||
@@ -55,6 +55,7 @@ from vllm.v1.engine import PauseMode
|
|||||||
from vllm.v1.engine.llm_engine import LLMEngine
|
from vllm.v1.engine.llm_engine import LLMEngine
|
||||||
from vllm.v1.sample.logits_processor import LogitsProcessor
|
from vllm.v1.sample.logits_processor import LogitsProcessor
|
||||||
|
|
||||||
|
from ..renderers import ChatParams
|
||||||
from .offline_utils import _O, _R, OfflineInferenceMixin
|
from .offline_utils import _O, _R, OfflineInferenceMixin
|
||||||
|
|
||||||
if TYPE_CHECKING:
|
if TYPE_CHECKING:
|
||||||
@@ -352,6 +353,8 @@ class LLM(BeamSearchOfflineMixin, PoolingOfflineMixin, OfflineInferenceMixin):
|
|||||||
self.chat_template = load_chat_template(chat_template)
|
self.chat_template = load_chat_template(chat_template)
|
||||||
self.input_processor = self.llm_engine.input_processor
|
self.input_processor = self.llm_engine.input_processor
|
||||||
|
|
||||||
|
self.renderer.warmup(ChatParams(chat_template=self.chat_template))
|
||||||
|
|
||||||
# The renderer thread pool is only consumed by the async renderer
|
# The renderer thread pool is only consumed by the async renderer
|
||||||
# path; the synchronous `LLM` entrypoint runs multimodal
|
# path; the synchronous `LLM` entrypoint runs multimodal
|
||||||
# preprocessing serially. Warn so the setting is not a silent
|
# preprocessing serially. Warn so the setting is not a silent
|
||||||
|
|||||||
@@ -426,6 +426,7 @@ async def init_app_state(
|
|||||||
default_chat_template_kwargs=args.default_chat_template_kwargs,
|
default_chat_template_kwargs=args.default_chat_template_kwargs,
|
||||||
log_error_stack=args.log_error_stack,
|
log_error_stack=args.log_error_stack,
|
||||||
)
|
)
|
||||||
|
state.online_renderer.warmup()
|
||||||
|
|
||||||
state.online_derenderer = OnlineDerenderer(
|
state.online_derenderer = OnlineDerenderer(
|
||||||
model_config=engine_client.model_config,
|
model_config=engine_client.model_config,
|
||||||
@@ -529,6 +530,7 @@ async def init_render_app_state(
|
|||||||
default_chat_template_kwargs=args.default_chat_template_kwargs,
|
default_chat_template_kwargs=args.default_chat_template_kwargs,
|
||||||
log_error_stack=args.log_error_stack,
|
log_error_stack=args.log_error_stack,
|
||||||
)
|
)
|
||||||
|
state.online_renderer.warmup()
|
||||||
|
|
||||||
state.online_derenderer = OnlineDerenderer(
|
state.online_derenderer = OnlineDerenderer(
|
||||||
model_config=vllm_config.model_config,
|
model_config=vllm_config.model_config,
|
||||||
|
|||||||
@@ -60,7 +60,6 @@ from vllm.logprobs import Logprob
|
|||||||
from vllm.outputs import RequestOutput
|
from vllm.outputs import RequestOutput
|
||||||
from vllm.parser import ParserManager
|
from vllm.parser import ParserManager
|
||||||
from vllm.parser.abstract_parser import Parser
|
from vllm.parser.abstract_parser import Parser
|
||||||
from vllm.renderers import ChatParams
|
|
||||||
from vllm.renderers.online_renderer import OnlineRenderer
|
from vllm.renderers.online_renderer import OnlineRenderer
|
||||||
from vllm.sampling_params import BeamSearchParams, SamplingParams
|
from vllm.sampling_params import BeamSearchParams, SamplingParams
|
||||||
from vllm.tokenizers import TokenizerLike
|
from vllm.tokenizers import TokenizerLike
|
||||||
@@ -178,15 +177,6 @@ class OpenAIServingChat(GenerateBaseServing):
|
|||||||
self.supports_code_interpreter = False
|
self.supports_code_interpreter = False
|
||||||
self.python_tool = None
|
self.python_tool = None
|
||||||
|
|
||||||
def warmup(self) -> None:
|
|
||||||
self.renderer.warmup(
|
|
||||||
ChatParams(
|
|
||||||
chat_template=self.chat_template,
|
|
||||||
chat_template_content_format=self.chat_template_content_format,
|
|
||||||
chat_template_kwargs=self.default_chat_template_kwargs,
|
|
||||||
)
|
|
||||||
)
|
|
||||||
|
|
||||||
def _effective_chat_template_kwargs(
|
def _effective_chat_template_kwargs(
|
||||||
self, request: ChatCompletionRequest
|
self, request: ChatCompletionRequest
|
||||||
) -> dict[str, Any]:
|
) -> dict[str, Any]:
|
||||||
|
|||||||
+35
-33
@@ -243,42 +243,44 @@ class BaseRenderer(ABC, Generic[_T]):
|
|||||||
"""
|
"""
|
||||||
from vllm.entrypoints.chat_utils import ChatTemplateResolutionError
|
from vllm.entrypoints.chat_utils import ChatTemplateResolutionError
|
||||||
|
|
||||||
try:
|
# prevent MM processor hangs
|
||||||
logger.debug("Warming up chat template processing...")
|
with set_default_torch_num_threads(1):
|
||||||
start_time = time.perf_counter()
|
|
||||||
|
|
||||||
self.render_chat([[{"role": "user", "content": "warmup"}]], chat_params)
|
|
||||||
|
|
||||||
elapsed = time.perf_counter() - start_time
|
|
||||||
logger.debug("Chat template warmup completed in %.3fs", elapsed)
|
|
||||||
except ChatTemplateResolutionError:
|
|
||||||
logger.debug("This model does not support chat template.")
|
|
||||||
except Exception:
|
|
||||||
logger.warning("Chat template warmup failed", exc_info=True)
|
|
||||||
|
|
||||||
if self.mm_processor:
|
|
||||||
try:
|
try:
|
||||||
logger.debug("Warming up multi-modal processing...")
|
logger.debug("Warming up chat template processing...")
|
||||||
self._warmup_mm_processor(
|
start_time = time.perf_counter()
|
||||||
self.mm_processor,
|
|
||||||
log_prefix="Multi-modal",
|
|
||||||
)
|
|
||||||
except Exception:
|
|
||||||
logger.warning("Multi-modal warmup failed")
|
|
||||||
finally:
|
|
||||||
self.clear_mm_cache()
|
|
||||||
|
|
||||||
if self._readonly_mm_processor is not None:
|
self.render_chat([[{"role": "user", "content": "warmup"}]], chat_params)
|
||||||
try:
|
|
||||||
logger.debug("Warming up readonly multi-modal processing...")
|
elapsed = time.perf_counter() - start_time
|
||||||
self._warmup_mm_processor(
|
logger.debug("Chat template warmup completed in %.3fs", elapsed)
|
||||||
self._readonly_mm_processor,
|
except ChatTemplateResolutionError:
|
||||||
log_prefix="Readonly multi-modal",
|
logger.debug("This model does not support chat template.")
|
||||||
)
|
|
||||||
except Exception:
|
except Exception:
|
||||||
logger.warning("Readonly multi-modal warmup failed")
|
logger.warning("Chat template warmup failed", exc_info=True)
|
||||||
finally:
|
|
||||||
self._clear_processor_cache(self._readonly_mm_processor)
|
if self.mm_processor:
|
||||||
|
try:
|
||||||
|
logger.debug("Warming up multi-modal processing...")
|
||||||
|
self._warmup_mm_processor(
|
||||||
|
self.mm_processor,
|
||||||
|
log_prefix="Multi-modal",
|
||||||
|
)
|
||||||
|
except Exception:
|
||||||
|
logger.warning("Multi-modal warmup failed")
|
||||||
|
finally:
|
||||||
|
self.clear_mm_cache()
|
||||||
|
|
||||||
|
if self._readonly_mm_processor is not None:
|
||||||
|
try:
|
||||||
|
logger.debug("Warming up readonly multi-modal processing...")
|
||||||
|
self._warmup_mm_processor(
|
||||||
|
self._readonly_mm_processor,
|
||||||
|
log_prefix="Readonly multi-modal",
|
||||||
|
)
|
||||||
|
except Exception:
|
||||||
|
logger.warning("Readonly multi-modal warmup failed")
|
||||||
|
finally:
|
||||||
|
self._clear_processor_cache(self._readonly_mm_processor)
|
||||||
|
|
||||||
async def clear_mm_cache_async(self) -> None:
|
async def clear_mm_cache_async(self) -> None:
|
||||||
"""Serialize clear_mm_cache through the multimodal executor to avoid
|
"""Serialize clear_mm_cache through the multimodal executor to avoid
|
||||||
|
|||||||
@@ -36,7 +36,7 @@ from vllm.inputs import (
|
|||||||
)
|
)
|
||||||
from vllm.logger import init_logger
|
from vllm.logger import init_logger
|
||||||
from vllm.parser import Parser, ParserManager
|
from vllm.parser import Parser, ParserManager
|
||||||
from vllm.renderers import BaseRenderer, merge_kwargs
|
from vllm.renderers import BaseRenderer, ChatParams, merge_kwargs
|
||||||
from vllm.renderers.inputs.preprocess import (
|
from vllm.renderers.inputs.preprocess import (
|
||||||
parse_model_prompt,
|
parse_model_prompt,
|
||||||
prompt_to_seq,
|
prompt_to_seq,
|
||||||
@@ -105,6 +105,15 @@ class OnlineRenderer:
|
|||||||
self.supports_browsing = False
|
self.supports_browsing = False
|
||||||
self.supports_code_interpreter = False
|
self.supports_code_interpreter = False
|
||||||
|
|
||||||
|
def warmup(self) -> None:
|
||||||
|
self.renderer.warmup(
|
||||||
|
ChatParams(
|
||||||
|
chat_template=self.chat_template,
|
||||||
|
chat_template_content_format=self.chat_template_content_format,
|
||||||
|
chat_template_kwargs=self.default_chat_template_kwargs,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
async def render_chat(
|
async def render_chat(
|
||||||
self,
|
self,
|
||||||
request: ChatCompletionRequest,
|
request: ChatCompletionRequest,
|
||||||
|
|||||||
Reference in New Issue
Block a user