[Bugfix] Offload blocking tokenizer ops to shared thread pool to unblock event loop (#34789)
Signed-off-by: Bvicii <yizhanhuang2002@gmail.com> Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
This commit is contained in:
+20
-9
@@ -30,6 +30,7 @@ from vllm.logger import init_logger
|
||||
from vllm.tokenizers.hf import HfTokenizer
|
||||
from vllm.transformers_utils.chat_templates import get_chat_template_fallback_path
|
||||
from vllm.transformers_utils.processor import cached_get_processor
|
||||
from vllm.utils.async_utils import make_async
|
||||
from vllm.utils.func_utils import supports_kw
|
||||
|
||||
from .base import BaseRenderer
|
||||
@@ -614,6 +615,10 @@ class HfRenderer(BaseRenderer[HfTokenizer]):
|
||||
config.model_config.hf_config, "use_unified_vision_chunk", False
|
||||
)
|
||||
|
||||
self._apply_chat_template_async = make_async(
|
||||
safe_apply_chat_template, executor=self._executor
|
||||
)
|
||||
|
||||
def render_messages(
|
||||
self,
|
||||
messages: list[ChatCompletionMessageParam],
|
||||
@@ -656,10 +661,13 @@ class HfRenderer(BaseRenderer[HfTokenizer]):
|
||||
video_placeholder = getattr(
|
||||
model_config.hf_config, "video_placeholder", None
|
||||
)
|
||||
prompt_raw = replace_vision_chunk_video_placeholder(
|
||||
prompt_raw,
|
||||
mm_data,
|
||||
video_placeholder,
|
||||
prompt_raw = cast(
|
||||
list[int],
|
||||
replace_vision_chunk_video_placeholder(
|
||||
prompt_raw,
|
||||
mm_data,
|
||||
video_placeholder,
|
||||
),
|
||||
)
|
||||
|
||||
prompt = parse_dec_only_prompt(prompt_raw)
|
||||
@@ -692,7 +700,7 @@ class HfRenderer(BaseRenderer[HfTokenizer]):
|
||||
mm_processor_kwargs=params.mm_processor_kwargs,
|
||||
)
|
||||
|
||||
prompt_raw = safe_apply_chat_template(
|
||||
prompt_raw = await self._apply_chat_template_async(
|
||||
model_config,
|
||||
tokenizer,
|
||||
conversation,
|
||||
@@ -710,10 +718,13 @@ class HfRenderer(BaseRenderer[HfTokenizer]):
|
||||
video_placeholder = getattr(
|
||||
model_config.hf_config, "video_placeholder", None
|
||||
)
|
||||
prompt_raw = replace_vision_chunk_video_placeholder(
|
||||
prompt_raw,
|
||||
mm_data,
|
||||
video_placeholder,
|
||||
prompt_raw = cast(
|
||||
list[int],
|
||||
replace_vision_chunk_video_placeholder(
|
||||
prompt_raw,
|
||||
mm_data,
|
||||
video_placeholder,
|
||||
),
|
||||
)
|
||||
|
||||
prompt = parse_dec_only_prompt(prompt_raw)
|
||||
|
||||
Reference in New Issue
Block a user