[Bugfix] Offload blocking tokenizer ops to shared thread pool to unblock event loop (#34789)

Signed-off-by: Bvicii <yizhanhuang2002@gmail.com>
Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
This commit is contained in:
Bvicii
2026-03-26 22:17:00 -07:00
committed by GitHub
co-authored by Cyrus Leung
parent d86060122a
commit 999dfc1622
15 changed files with 195 additions and 28 deletions
+20 -9
View File
@@ -30,6 +30,7 @@ from vllm.logger import init_logger
from vllm.tokenizers.hf import HfTokenizer
from vllm.transformers_utils.chat_templates import get_chat_template_fallback_path
from vllm.transformers_utils.processor import cached_get_processor
from vllm.utils.async_utils import make_async
from vllm.utils.func_utils import supports_kw
from .base import BaseRenderer
@@ -614,6 +615,10 @@ class HfRenderer(BaseRenderer[HfTokenizer]):
config.model_config.hf_config, "use_unified_vision_chunk", False
)
self._apply_chat_template_async = make_async(
safe_apply_chat_template, executor=self._executor
)
def render_messages(
self,
messages: list[ChatCompletionMessageParam],
@@ -656,10 +661,13 @@ class HfRenderer(BaseRenderer[HfTokenizer]):
video_placeholder = getattr(
model_config.hf_config, "video_placeholder", None
)
prompt_raw = replace_vision_chunk_video_placeholder(
prompt_raw,
mm_data,
video_placeholder,
prompt_raw = cast(
list[int],
replace_vision_chunk_video_placeholder(
prompt_raw,
mm_data,
video_placeholder,
),
)
prompt = parse_dec_only_prompt(prompt_raw)
@@ -692,7 +700,7 @@ class HfRenderer(BaseRenderer[HfTokenizer]):
mm_processor_kwargs=params.mm_processor_kwargs,
)
prompt_raw = safe_apply_chat_template(
prompt_raw = await self._apply_chat_template_async(
model_config,
tokenizer,
conversation,
@@ -710,10 +718,13 @@ class HfRenderer(BaseRenderer[HfTokenizer]):
video_placeholder = getattr(
model_config.hf_config, "video_placeholder", None
)
prompt_raw = replace_vision_chunk_video_placeholder(
prompt_raw,
mm_data,
video_placeholder,
prompt_raw = cast(
list[int],
replace_vision_chunk_video_placeholder(
prompt_raw,
mm_data,
video_placeholder,
),
)
prompt = parse_dec_only_prompt(prompt_raw)