[Core] Offload raw-prompt preprocessing to renderer thread pool in AsyncLLM (#49608)
Signed-off-by: almogtavor <almogtavor@gmail.com>
This commit is contained in:
@@ -28,6 +28,7 @@ from vllm.sampling_params import SamplingParams
|
||||
from vllm.tasks import GENERATION_TASKS, POOLING_TASKS, SupportedTask
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
from vllm.utils import length_from_prompt_token_ids_or_embeds, random_uuid
|
||||
from vllm.utils.async_utils import make_async
|
||||
from vllm.utils.jsontree import json_iter_leaves
|
||||
from vllm.v1.engine import EngineCoreRequest
|
||||
|
||||
@@ -73,6 +74,13 @@ class InputProcessor:
|
||||
mm_registry=mm_registry,
|
||||
)
|
||||
|
||||
# Raw-prompt preprocessing (tokenization and multimodal processing)
|
||||
# is blocking, so async callers should run it on the renderer's
|
||||
# thread pool to keep their event loop responsive.
|
||||
self.process_inputs_async = make_async(
|
||||
self.process_inputs, executor=self.renderer._executor
|
||||
)
|
||||
|
||||
@property
|
||||
def tokenizer(self) -> TokenizerLike | None:
|
||||
return self.renderer.tokenizer
|
||||
|
||||
Reference in New Issue
Block a user