[Bugfix] Offload blocking tokenizer ops to shared thread pool to unblock event loop (#34789)
Signed-off-by: Bvicii <yizhanhuang2002@gmail.com> Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
This commit is contained in:
+118
-6
@@ -5,6 +5,7 @@ import copy
|
||||
import time
|
||||
from abc import ABC, abstractmethod
|
||||
from collections.abc import Mapping, Sequence
|
||||
from concurrent.futures import Executor, ThreadPoolExecutor
|
||||
from functools import cached_property
|
||||
from typing import TYPE_CHECKING, Any, Generic, overload
|
||||
|
||||
@@ -38,7 +39,10 @@ from vllm.multimodal.processing import BaseMultiModalProcessor
|
||||
from vllm.multimodal.processing import ProcessorInputs as MMProcessorInputs
|
||||
from vllm.multimodal.registry import MultiModalTimingRegistry
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
from vllm.utils.async_utils import AsyncMicrobatchTokenizer
|
||||
from vllm.utils.async_utils import (
|
||||
AsyncMicrobatchTokenizer,
|
||||
make_async,
|
||||
)
|
||||
from vllm.utils.counter import AtomicCounter
|
||||
from vllm.utils.torch_utils import set_default_torch_num_threads
|
||||
from vllm.v1.metrics.stats import MultiModalCacheStats
|
||||
@@ -78,11 +82,28 @@ class BaseRenderer(ABC, Generic[_T]):
|
||||
|
||||
self.tokenizer = tokenizer
|
||||
|
||||
# Shared thread pool executor for blocking tokenizer and
|
||||
# multimodal preprocessing operations. The multimodal processor
|
||||
# receives a deep-copied tokenizer (see #36557) so it is safe to
|
||||
# run tokenization and MM preprocessing concurrently.
|
||||
pool_workers = config.model_config.renderer_num_workers
|
||||
self._executor = ThreadPoolExecutor(max_workers=pool_workers)
|
||||
|
||||
# Multimodal preprocessing is always offloaded to the thread pool
|
||||
# to keep the asyncio event loop responsive under concurrent load.
|
||||
self._mm_executor: Executor = self._executor
|
||||
|
||||
# Lazy initialization since offline LLM doesn't use async
|
||||
self._async_tokenizer: AsyncMicrobatchTokenizer | None = None
|
||||
|
||||
self.mm_processor: BaseMultiModalProcessor | None = None
|
||||
self._mm_cache_stats: MultiModalCacheStats | None = None
|
||||
self._clear_mm_cache_async = make_async(
|
||||
self.clear_mm_cache, executor=self._executor
|
||||
)
|
||||
self._process_multimodal_async = make_async(
|
||||
self._process_multimodal, executor=self._mm_executor
|
||||
)
|
||||
if config.model_config.is_multimodal_model:
|
||||
mm_processor_cache = mm_registry.processor_cache_from_config(config)
|
||||
|
||||
@@ -119,7 +140,9 @@ class BaseRenderer(ABC, Generic[_T]):
|
||||
|
||||
def get_async_tokenizer(self) -> AsyncMicrobatchTokenizer:
|
||||
if self._async_tokenizer is None:
|
||||
self._async_tokenizer = AsyncMicrobatchTokenizer(self.get_tokenizer())
|
||||
self._async_tokenizer = AsyncMicrobatchTokenizer(
|
||||
self.get_tokenizer(), executor=self._executor
|
||||
)
|
||||
|
||||
return self._async_tokenizer
|
||||
|
||||
@@ -211,11 +234,24 @@ class BaseRenderer(ABC, Generic[_T]):
|
||||
finally:
|
||||
self.clear_mm_cache()
|
||||
|
||||
async def clear_mm_cache_async(self) -> None:
|
||||
"""Serialize clear_mm_cache through the shared executor to avoid
|
||||
races with concurrent process_inputs on the mm_processor_cache."""
|
||||
await self._clear_mm_cache_async()
|
||||
|
||||
def shutdown(self) -> None:
|
||||
mm_processor_cache = self.mm_processor_cache
|
||||
if mm_processor_cache is not None:
|
||||
mm_processor_cache.close()
|
||||
|
||||
if executor := getattr(self, "_executor", None):
|
||||
executor.shutdown(wait=False)
|
||||
|
||||
if (
|
||||
mm_executor := getattr(self, "_mm_executor", None)
|
||||
) is not None and mm_executor is not executor:
|
||||
mm_executor.shutdown(wait=False)
|
||||
|
||||
def get_bos_token_id(self) -> int | None:
|
||||
if self.tokenizer is None:
|
||||
logger.warning_once(
|
||||
@@ -621,6 +657,9 @@ class BaseRenderer(ABC, Generic[_T]):
|
||||
self,
|
||||
prompt: TokensPrompt,
|
||||
) -> TokensInput | MultiModalInput:
|
||||
"""Process token inputs, with multimodal preprocessing offloaded
|
||||
to the shared thread pool in the async variant.
|
||||
"""
|
||||
prompt_token_ids = prompt["prompt_token_ids"]
|
||||
|
||||
engine_input: TokensInput | MultiModalInput
|
||||
@@ -670,12 +709,46 @@ class BaseRenderer(ABC, Generic[_T]):
|
||||
cache_salt=prompt.get("cache_salt"),
|
||||
)
|
||||
|
||||
async def _process_tokens_async(
|
||||
self,
|
||||
prompt: TokensPrompt,
|
||||
) -> TokensInput | MultiModalInput:
|
||||
prompt_token_ids = prompt["prompt_token_ids"]
|
||||
|
||||
engine_input: TokensInput | MultiModalInput
|
||||
if multi_modal_data := prompt.get("multi_modal_data"):
|
||||
engine_input = await self._process_multimodal_async(
|
||||
prompt_token_ids,
|
||||
multi_modal_data,
|
||||
mm_processor_kwargs=prompt.get("mm_processor_kwargs"),
|
||||
tokenization_kwargs=None,
|
||||
mm_uuids=prompt.get("multi_modal_uuids"),
|
||||
)
|
||||
else:
|
||||
engine_input = tokens_input(prompt_token_ids)
|
||||
|
||||
if prompt_text := prompt.get("prompt"):
|
||||
engine_input["prompt"] = prompt_text
|
||||
if cache_salt := prompt.get("cache_salt"):
|
||||
engine_input["cache_salt"] = cache_salt
|
||||
|
||||
return engine_input
|
||||
|
||||
def _process_singleton(self, prompt: SingletonTokPrompt) -> SingletonInput:
|
||||
if "prompt_embeds" in prompt:
|
||||
return self._process_embeds(prompt) # type: ignore[arg-type]
|
||||
|
||||
return self._process_tokens(prompt) # type: ignore[arg-type]
|
||||
|
||||
async def _process_singleton_async(
|
||||
self,
|
||||
prompt: SingletonTokPrompt,
|
||||
) -> SingletonInput:
|
||||
if "prompt_embeds" in prompt:
|
||||
return self._process_embeds(prompt) # type: ignore[arg-type]
|
||||
|
||||
return await self._process_tokens_async(prompt) # type: ignore[arg-type]
|
||||
|
||||
def _process_enc_dec(
|
||||
self,
|
||||
prompt: EncoderDecoderTokPrompt,
|
||||
@@ -699,6 +772,28 @@ class BaseRenderer(ABC, Generic[_T]):
|
||||
skip_decoder_start_token=skip_decoder_start_token,
|
||||
)
|
||||
|
||||
async def _process_enc_dec_async(
|
||||
self,
|
||||
prompt: EncoderDecoderTokPrompt,
|
||||
) -> EncoderDecoderInput:
|
||||
enc_prompt = prompt["encoder_prompt"]
|
||||
dec_prompt = prompt["decoder_prompt"]
|
||||
|
||||
encoder_input, decoder_input = await asyncio.gather(
|
||||
self._process_singleton_async(enc_prompt),
|
||||
(
|
||||
asyncio.sleep(0)
|
||||
if dec_prompt is None
|
||||
else self._process_singleton_async(dec_prompt)
|
||||
),
|
||||
)
|
||||
|
||||
return build_enc_dec_input(
|
||||
encoder_input=encoder_input,
|
||||
decoder_input=decoder_input,
|
||||
decoder_start_token_id=self.get_dec_start_token_id(),
|
||||
)
|
||||
|
||||
def process_for_engine(self, prompt: TokPrompt, arrival_time: float) -> EngineInput:
|
||||
engine_input: EngineInput
|
||||
if "encoder_prompt" in prompt:
|
||||
@@ -710,6 +805,21 @@ class BaseRenderer(ABC, Generic[_T]):
|
||||
|
||||
return engine_input
|
||||
|
||||
async def process_for_engine_async(
|
||||
self, prompt: TokPrompt, arrival_time: float
|
||||
) -> EngineInput:
|
||||
engine_input: EngineInput
|
||||
if "encoder_prompt" in prompt:
|
||||
engine_input = await self._process_enc_dec_async(
|
||||
prompt # type: ignore[arg-type]
|
||||
)
|
||||
else:
|
||||
engine_input = await self._process_singleton_async(prompt)
|
||||
|
||||
engine_input["arrival_time"] = arrival_time
|
||||
|
||||
return engine_input
|
||||
|
||||
# Top-level methods
|
||||
def render_cmpl(
|
||||
self,
|
||||
@@ -747,7 +857,9 @@ class BaseRenderer(ABC, Generic[_T]):
|
||||
|
||||
self._apply_prompt_extras(tok_prompts, prompt_extras)
|
||||
|
||||
return [self.process_for_engine(prompt, arrival_time) for prompt in tok_prompts]
|
||||
return await asyncio.gather(
|
||||
*(self.process_for_engine_async(p, arrival_time) for p in tok_prompts)
|
||||
)
|
||||
|
||||
def render_chat(
|
||||
self,
|
||||
@@ -811,8 +923,8 @@ class BaseRenderer(ABC, Generic[_T]):
|
||||
|
||||
self._apply_prompt_extras(tok_prompts, prompt_extras)
|
||||
|
||||
eng_prompts = [
|
||||
self.process_for_engine(prompt, arrival_time) for prompt in tok_prompts
|
||||
]
|
||||
eng_prompts = await asyncio.gather(
|
||||
*(self.process_for_engine_async(p, arrival_time) for p in tok_prompts)
|
||||
)
|
||||
|
||||
return out_conversations, eng_prompts
|
||||
|
||||
Reference in New Issue
Block a user