[Bugfix] Warn when renderer_num_workers has no effect on offline LLM (#42905)
Signed-off-by: Daoyuan Li <94409450+DaoyuanLi2816@users.noreply.github.com>
This commit is contained in:
@@ -329,9 +329,15 @@ class ModelConfig:
|
||||
io_processor_plugin: str | None = None
|
||||
"""IOProcessor plugin name to load at model startup"""
|
||||
renderer_num_workers: int = 1
|
||||
"""Number of worker threads in the renderer thread pool. This pool
|
||||
handles async tokenization, chat template rendering, and multimodal
|
||||
preprocessing."""
|
||||
"""Number of worker threads in the renderer thread pool. The pool is
|
||||
consumed by the async renderer path (e.g. the OpenAI-compatible API
|
||||
server started by `vllm serve`) to parallelize tokenization, chat
|
||||
template rendering, and multimodal preprocessing across concurrent
|
||||
requests.
|
||||
|
||||
The offline `LLM` entrypoint uses the synchronous renderer path and
|
||||
processes prompts (including multimodal preprocessing) serially, so
|
||||
this setting has no effect there."""
|
||||
|
||||
# Pooler config
|
||||
pooler_config: PoolerConfig | None = None
|
||||
|
||||
@@ -381,6 +381,21 @@ class LLM(BeamSearchOfflineMixin, PoolingOfflineMixin):
|
||||
self.chat_template = load_chat_template(chat_template)
|
||||
self.input_processor = self.llm_engine.input_processor
|
||||
|
||||
# The renderer thread pool is only consumed by the async renderer
|
||||
# path; the synchronous `LLM` entrypoint runs multimodal
|
||||
# preprocessing serially. Warn so the setting is not a silent
|
||||
# no-op. See vllm-project/vllm#42901.
|
||||
if self.model_config.renderer_num_workers > 1:
|
||||
logger.warning_once(
|
||||
"`renderer_num_workers=%d` was set, but the offline `LLM` "
|
||||
"entrypoint uses the synchronous renderer path and runs "
|
||||
"multimodal preprocessing serially across prompts. The "
|
||||
"renderer thread pool is only consumed by the async "
|
||||
"renderer path used by `vllm serve` / `AsyncLLM`, so this "
|
||||
"setting has no effect here.",
|
||||
self.model_config.renderer_num_workers,
|
||||
)
|
||||
|
||||
PoolingOfflineMixin.__init__(self)
|
||||
|
||||
# Cache for __repr__ to avoid repeated collective_rpc calls
|
||||
|
||||
Reference in New Issue
Block a user