[Core] Offload raw-prompt preprocessing to renderer thread pool in AsyncLLM (#49608)
Signed-off-by: almogtavor <almogtavor@gmail.com>
This commit is contained in:
+29
-12
@@ -349,18 +349,35 @@ class AsyncLLM(EngineClient):
|
||||
"latter will be used, and the former will be ignored."
|
||||
)
|
||||
else:
|
||||
request = self.input_processor.process_inputs(
|
||||
request_id,
|
||||
prompt,
|
||||
params,
|
||||
supported_tasks=await self.get_supported_tasks(),
|
||||
arrival_time=arrival_time,
|
||||
lora_request=lora_request,
|
||||
tokenization_kwargs=tokenization_kwargs,
|
||||
trace_headers=trace_headers,
|
||||
priority=priority,
|
||||
data_parallel_rank=data_parallel_rank,
|
||||
)
|
||||
if isinstance(prompt, dict) and "type" in prompt:
|
||||
# Rendered EngineInput; no blocking preprocessing needed.
|
||||
request = self.input_processor.process_inputs(
|
||||
request_id,
|
||||
prompt,
|
||||
params,
|
||||
supported_tasks=await self.get_supported_tasks(),
|
||||
arrival_time=arrival_time,
|
||||
lora_request=lora_request,
|
||||
tokenization_kwargs=tokenization_kwargs,
|
||||
trace_headers=trace_headers,
|
||||
priority=priority,
|
||||
data_parallel_rank=data_parallel_rank,
|
||||
)
|
||||
else:
|
||||
# Raw prompts require tokenization and possibly multimodal
|
||||
# processing, which must not block the event loop.
|
||||
request = await self.input_processor.process_inputs_async(
|
||||
request_id,
|
||||
prompt,
|
||||
params,
|
||||
supported_tasks=await self.get_supported_tasks(),
|
||||
arrival_time=arrival_time,
|
||||
lora_request=lora_request,
|
||||
tokenization_kwargs=tokenization_kwargs,
|
||||
trace_headers=trace_headers,
|
||||
priority=priority,
|
||||
data_parallel_rank=data_parallel_rank,
|
||||
)
|
||||
prompt_text, _, _ = extract_prompt_components(self.model_config, prompt)
|
||||
|
||||
if reasoning_ended is not None:
|
||||
|
||||
@@ -28,6 +28,7 @@ from vllm.sampling_params import SamplingParams
|
||||
from vllm.tasks import GENERATION_TASKS, POOLING_TASKS, SupportedTask
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
from vllm.utils import length_from_prompt_token_ids_or_embeds, random_uuid
|
||||
from vllm.utils.async_utils import make_async
|
||||
from vllm.utils.jsontree import json_iter_leaves
|
||||
from vllm.v1.engine import EngineCoreRequest
|
||||
|
||||
@@ -73,6 +74,13 @@ class InputProcessor:
|
||||
mm_registry=mm_registry,
|
||||
)
|
||||
|
||||
# Raw-prompt preprocessing (tokenization and multimodal processing)
|
||||
# is blocking, so async callers should run it on the renderer's
|
||||
# thread pool to keep their event loop responsive.
|
||||
self.process_inputs_async = make_async(
|
||||
self.process_inputs, executor=self.renderer._executor
|
||||
)
|
||||
|
||||
@property
|
||||
def tokenizer(self) -> TokenizerLike | None:
|
||||
return self.renderer.tokenizer
|
||||
|
||||
Reference in New Issue
Block a user