From 39f55ffdaacbf4f2ac06e395c3f9c6e55a54dbd0 Mon Sep 17 00:00:00 2001 From: Almog Tavor <70065337+almogtavor@users.noreply.github.com> Date: Sat, 1 Aug 2026 18:10:51 +0300 Subject: [PATCH] [Core] Offload raw-prompt preprocessing to renderer thread pool in AsyncLLM (#49608) Signed-off-by: almogtavor --- vllm/v1/engine/async_llm.py | 41 ++++++++++++++++++++++--------- vllm/v1/engine/input_processor.py | 8 ++++++ 2 files changed, 37 insertions(+), 12 deletions(-) diff --git a/vllm/v1/engine/async_llm.py b/vllm/v1/engine/async_llm.py index 1718a5adc3..4ce9d64b44 100644 --- a/vllm/v1/engine/async_llm.py +++ b/vllm/v1/engine/async_llm.py @@ -349,18 +349,35 @@ class AsyncLLM(EngineClient): "latter will be used, and the former will be ignored." ) else: - request = self.input_processor.process_inputs( - request_id, - prompt, - params, - supported_tasks=await self.get_supported_tasks(), - arrival_time=arrival_time, - lora_request=lora_request, - tokenization_kwargs=tokenization_kwargs, - trace_headers=trace_headers, - priority=priority, - data_parallel_rank=data_parallel_rank, - ) + if isinstance(prompt, dict) and "type" in prompt: + # Rendered EngineInput; no blocking preprocessing needed. + request = self.input_processor.process_inputs( + request_id, + prompt, + params, + supported_tasks=await self.get_supported_tasks(), + arrival_time=arrival_time, + lora_request=lora_request, + tokenization_kwargs=tokenization_kwargs, + trace_headers=trace_headers, + priority=priority, + data_parallel_rank=data_parallel_rank, + ) + else: + # Raw prompts require tokenization and possibly multimodal + # processing, which must not block the event loop. + request = await self.input_processor.process_inputs_async( + request_id, + prompt, + params, + supported_tasks=await self.get_supported_tasks(), + arrival_time=arrival_time, + lora_request=lora_request, + tokenization_kwargs=tokenization_kwargs, + trace_headers=trace_headers, + priority=priority, + data_parallel_rank=data_parallel_rank, + ) prompt_text, _, _ = extract_prompt_components(self.model_config, prompt) if reasoning_ended is not None: diff --git a/vllm/v1/engine/input_processor.py b/vllm/v1/engine/input_processor.py index 735593672d..1fe460b54a 100644 --- a/vllm/v1/engine/input_processor.py +++ b/vllm/v1/engine/input_processor.py @@ -28,6 +28,7 @@ from vllm.sampling_params import SamplingParams from vllm.tasks import GENERATION_TASKS, POOLING_TASKS, SupportedTask from vllm.tokenizers import TokenizerLike from vllm.utils import length_from_prompt_token_ids_or_embeds, random_uuid +from vllm.utils.async_utils import make_async from vllm.utils.jsontree import json_iter_leaves from vllm.v1.engine import EngineCoreRequest @@ -73,6 +74,13 @@ class InputProcessor: mm_registry=mm_registry, ) + # Raw-prompt preprocessing (tokenization and multimodal processing) + # is blocking, so async callers should run it on the renderer's + # thread pool to keep their event loop responsive. + self.process_inputs_async = make_async( + self.process_inputs, executor=self.renderer._executor + ) + @property def tokenizer(self) -> TokenizerLike | None: return self.renderer.tokenizer