[Bugfix] Offload blocking tokenizer ops to shared thread pool to unblock event loop (#34789)

Signed-off-by: Bvicii <yizhanhuang2002@gmail.com>
Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
This commit is contained in:
Bvicii
2026-03-26 22:17:00 -07:00
committed by GitHub
co-authored by Cyrus Leung
parent d86060122a
commit 999dfc1622
15 changed files with 195 additions and 28 deletions
+18 -4
View File
@@ -1,6 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from vllm.config import VllmConfig
from vllm.entrypoints.chat_utils import (
ChatCompletionMessageParam,
ConversationMessage,
@@ -9,6 +10,7 @@ from vllm.entrypoints.chat_utils import (
)
from vllm.logger import init_logger
from vllm.tokenizers.deepseek_v32 import DeepseekV32Tokenizer
from vllm.utils.async_utils import make_async
from .base import BaseRenderer
from .inputs import DictPrompt
@@ -19,12 +21,25 @@ logger = init_logger(__name__)
class DeepseekV32Renderer(BaseRenderer[DeepseekV32Tokenizer]):
def __init__(
self,
config: VllmConfig,
tokenizer: DeepseekV32Tokenizer | None,
) -> None:
super().__init__(config, tokenizer)
self._apply_chat_template_async = make_async(
self._apply_chat_template, executor=self._executor
)
def _apply_chat_template(self, *args, **kwargs):
return self.get_tokenizer().apply_chat_template(*args, **kwargs)
def render_messages(
self,
messages: list[ChatCompletionMessageParam],
params: ChatParams,
) -> tuple[list[ConversationMessage], DictPrompt]:
tokenizer = self.get_tokenizer()
conversation, mm_data, mm_uuids = parse_chat_messages(
messages,
self.model_config,
@@ -33,7 +48,7 @@ class DeepseekV32Renderer(BaseRenderer[DeepseekV32Tokenizer]):
mm_processor_kwargs=params.mm_processor_kwargs,
)
prompt_raw = tokenizer.apply_chat_template(
prompt_raw = self._apply_chat_template(
conversation=conversation,
messages=messages,
**params.get_apply_chat_template_kwargs(),
@@ -52,7 +67,6 @@ class DeepseekV32Renderer(BaseRenderer[DeepseekV32Tokenizer]):
messages: list[ChatCompletionMessageParam],
params: ChatParams,
) -> tuple[list[ConversationMessage], DictPrompt]:
tokenizer = self.get_tokenizer()
conversation, mm_data, mm_uuids = await parse_chat_messages_async(
messages,
self.model_config,
@@ -61,7 +75,7 @@ class DeepseekV32Renderer(BaseRenderer[DeepseekV32Tokenizer]):
mm_processor_kwargs=params.mm_processor_kwargs,
)
prompt_raw = tokenizer.apply_chat_template(
prompt_raw = await self._apply_chat_template_async(
conversation=conversation,
messages=messages,
**params.get_apply_chat_template_kwargs(),