[Core] Offload raw-prompt preprocessing to renderer thread pool in AsyncLLM (#49608)

Signed-off-by: almogtavor <almogtavor@gmail.com>
This commit is contained in:
Almog Tavor
2026-08-01 15:10:51 +00:00
committed by GitHub
parent 63e78ce365
commit 39f55ffdaa
2 changed files with 37 additions and 12 deletions
+8
View File
@@ -28,6 +28,7 @@ from vllm.sampling_params import SamplingParams
from vllm.tasks import GENERATION_TASKS, POOLING_TASKS, SupportedTask
from vllm.tokenizers import TokenizerLike
from vllm.utils import length_from_prompt_token_ids_or_embeds, random_uuid
from vllm.utils.async_utils import make_async
from vllm.utils.jsontree import json_iter_leaves
from vllm.v1.engine import EngineCoreRequest
@@ -73,6 +74,13 @@ class InputProcessor:
mm_registry=mm_registry,
)
# Raw-prompt preprocessing (tokenization and multimodal processing)
# is blocking, so async callers should run it on the renderer's
# thread pool to keep their event loop responsive.
self.process_inputs_async = make_async(
self.process_inputs, executor=self.renderer._executor
)
@property
def tokenizer(self) -> TokenizerLike | None:
return self.renderer.tokenizer