[Core] Offload raw-prompt preprocessing to renderer thread pool in AsyncLLM (#49608)

Signed-off-by: almogtavor <almogtavor@gmail.com>
This commit is contained in:
Almog Tavor
2026-08-01 15:10:51 +00:00
committed by GitHub
parent 63e78ce365
commit 39f55ffdaa
2 changed files with 37 additions and 12 deletions
+29 -12
View File
@@ -349,18 +349,35 @@ class AsyncLLM(EngineClient):
"latter will be used, and the former will be ignored."
)
else:
request = self.input_processor.process_inputs(
request_id,
prompt,
params,
supported_tasks=await self.get_supported_tasks(),
arrival_time=arrival_time,
lora_request=lora_request,
tokenization_kwargs=tokenization_kwargs,
trace_headers=trace_headers,
priority=priority,
data_parallel_rank=data_parallel_rank,
)
if isinstance(prompt, dict) and "type" in prompt:
# Rendered EngineInput; no blocking preprocessing needed.
request = self.input_processor.process_inputs(
request_id,
prompt,
params,
supported_tasks=await self.get_supported_tasks(),
arrival_time=arrival_time,
lora_request=lora_request,
tokenization_kwargs=tokenization_kwargs,
trace_headers=trace_headers,
priority=priority,
data_parallel_rank=data_parallel_rank,
)
else:
# Raw prompts require tokenization and possibly multimodal
# processing, which must not block the event loop.
request = await self.input_processor.process_inputs_async(
request_id,
prompt,
params,
supported_tasks=await self.get_supported_tasks(),
arrival_time=arrival_time,
lora_request=lora_request,
tokenization_kwargs=tokenization_kwargs,
trace_headers=trace_headers,
priority=priority,
data_parallel_rank=data_parallel_rank,
)
prompt_text, _, _ = extract_prompt_components(self.model_config, prompt)
if reasoning_ended is not None:
+8
View File
@@ -28,6 +28,7 @@ from vllm.sampling_params import SamplingParams
from vllm.tasks import GENERATION_TASKS, POOLING_TASKS, SupportedTask
from vllm.tokenizers import TokenizerLike
from vllm.utils import length_from_prompt_token_ids_or_embeds, random_uuid
from vllm.utils.async_utils import make_async
from vllm.utils.jsontree import json_iter_leaves
from vllm.v1.engine import EngineCoreRequest
@@ -73,6 +74,13 @@ class InputProcessor:
mm_registry=mm_registry,
)
# Raw-prompt preprocessing (tokenization and multimodal processing)
# is blocking, so async callers should run it on the renderer's
# thread pool to keep their event loop responsive.
self.process_inputs_async = make_async(
self.process_inputs, executor=self.renderer._executor
)
@property
def tokenizer(self) -> TokenizerLike | None:
return self.renderer.tokenizer