[Misc] Reorganize inputs (#35182)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
Cyrus Leung
2026-03-25 10:22:54 -07:00
committed by GitHub
parent 678b3c99e8
commit ba2f0acc2d
142 changed files with 1212 additions and 1342 deletions
+38 -37
View File
@@ -34,9 +34,15 @@ from vllm.entrypoints.utils import (
create_error_response,
get_max_tokens,
)
from vllm.inputs.data import ProcessorInputs, PromptType, SingletonPrompt, TokensPrompt
from vllm.inputs import (
EngineInput,
MultiModalHashes,
MultiModalPlaceholders,
PromptType,
SingletonPrompt,
tokens_input,
)
from vllm.logger import init_logger
from vllm.multimodal.inputs import MultiModalHashes, MultiModalPlaceholderDict
from vllm.parser import ParserManager
from vllm.renderers import BaseRenderer, merge_kwargs
from vllm.renderers.inputs.preprocess import (
@@ -127,22 +133,22 @@ class OpenAIServingRender:
if isinstance(result, ErrorResponse):
return result
_, engine_prompts = result
_, engine_inputs = result
if len(engine_prompts) != 1:
if len(engine_inputs) != 1:
return self.create_error_response(
f"Expected exactly 1 engine prompt, got {len(engine_prompts)}"
f"Expected exactly 1 engine prompt, got {len(engine_inputs)}"
)
engine_prompt = engine_prompts[0]
engine_input = engine_inputs[0]
prompt_components = extract_prompt_components(self.model_config, engine_prompt)
prompt_components = extract_prompt_components(self.model_config, engine_input)
token_ids = prompt_components.token_ids
if not token_ids:
return self.create_error_response("No token_ids rendered")
token_ids = list(token_ids)
input_length = extract_prompt_len(self.model_config, engine_prompt)
input_length = extract_prompt_len(self.model_config, engine_input)
max_tokens = get_max_tokens(
self.model_config.max_model_len,
request.max_completion_tokens
@@ -159,7 +165,7 @@ class OpenAIServingRender:
return GenerateRequest(
request_id=request_id,
token_ids=token_ids,
features=self._extract_mm_features(engine_prompt),
features=self._extract_mm_features(engine_input),
sampling_params=params,
model=request.model,
stream=bool(request.stream),
@@ -171,7 +177,7 @@ class OpenAIServingRender:
async def render_chat(
self,
request: ChatCompletionRequest,
) -> tuple[list[ConversationMessage], list[ProcessorInputs]] | ErrorResponse:
) -> tuple[list[ConversationMessage], list[EngineInput]] | ErrorResponse:
"""Core preprocessing logic for chat requests (no model/engine check).
Called directly by render_chat_request and delegated to by
@@ -184,7 +190,6 @@ class OpenAIServingRender:
if is_mistral_tokenizer(tokenizer):
# because of issues with pydantic we need to potentially
# re-serialize the tool_calls field of the request
# for more info: see comment in `maybe_serialize_tool_calls`
_mt.maybe_serialize_tool_calls(request) # type: ignore[arg-type]
_mt.truncate_tool_call_ids(request) # type: ignore[arg-type]
_mt.validate_request_params(request)
@@ -232,7 +237,7 @@ class OpenAIServingRender:
if error_check_ret is not None:
return error_check_ret
conversation, engine_prompts = await self.preprocess_chat(
conversation, engine_inputs = await self.preprocess_chat(
request,
request.messages,
default_template=self.chat_template,
@@ -244,11 +249,11 @@ class OpenAIServingRender:
else:
# For GPT-OSS.
should_include_tools = tool_dicts is not None
conversation, engine_prompts = self._make_request_with_harmony(
conversation, engine_inputs = self._make_request_with_harmony(
request, should_include_tools
)
return conversation, engine_prompts
return conversation, engine_inputs
async def render_completion_request(
self,
@@ -266,16 +271,16 @@ class OpenAIServingRender:
if isinstance(result, ErrorResponse):
return result
generate_requests: list[GenerateRequest] = []
for engine_prompt in result:
for engine_input in result:
prompt_components = extract_prompt_components(
self.model_config, engine_prompt
self.model_config, engine_input
)
token_ids = prompt_components.token_ids
if not token_ids:
return self.create_error_response("No token_ids rendered")
token_ids = list(token_ids)
input_length = extract_prompt_len(self.model_config, engine_prompt)
input_length = extract_prompt_len(self.model_config, engine_input)
max_tokens = get_max_tokens(
self.model_config.max_model_len,
request.max_tokens,
@@ -293,7 +298,7 @@ class OpenAIServingRender:
GenerateRequest(
request_id=request_id,
token_ids=token_ids,
features=self._extract_mm_features(engine_prompt),
features=self._extract_mm_features(engine_input),
sampling_params=params,
model=request.model,
stream=bool(request.stream),
@@ -308,7 +313,7 @@ class OpenAIServingRender:
async def render_completion(
self,
request: CompletionRequest,
) -> list[ProcessorInputs] | ErrorResponse:
) -> list[EngineInput] | ErrorResponse:
"""Core preprocessing logic for completion requests (no model/engine check).
Called directly by render_completion_request and delegated to by
@@ -326,28 +331,28 @@ class OpenAIServingRender:
"prompt_logprobs is not compatible with prompt embeds."
)
engine_prompts = await self.preprocess_completion(
engine_inputs = await self.preprocess_completion(
request,
prompt_input=request.prompt,
prompt_embeds=request.prompt_embeds,
)
return engine_prompts
return engine_inputs
@staticmethod
def _extract_mm_features(
engine_prompt: ProcessorInputs,
engine_input: EngineInput,
) -> MultiModalFeatures | None:
"""Extract multimodal metadata from a rendered engine prompt.
Returns ``None`` for text-only prompts.
"""
if engine_prompt.get("type") != "multimodal":
if engine_input.get("type") != "multimodal":
return None
# At this point engine_prompt is a MultiModalInputs TypedDict.
mm_hashes: MultiModalHashes = engine_prompt["mm_hashes"] # type: ignore[typeddict-item]
raw_placeholders: MultiModalPlaceholderDict = engine_prompt["mm_placeholders"] # type: ignore[typeddict-item]
# At this point engine_input is a MultiModalInputs TypedDict.
mm_hashes: MultiModalHashes = engine_input["mm_hashes"] # type: ignore[typeddict-item]
raw_placeholders: MultiModalPlaceholders = engine_input["mm_placeholders"] # type: ignore[typeddict-item]
mm_placeholders = {
modality: [
@@ -401,13 +406,9 @@ class OpenAIServingRender:
# Render prompt token ids.
prompt_token_ids = render_for_completion(messages)
engine_prompt = TokensPrompt(prompt_token_ids=prompt_token_ids)
engine_input = tokens_input(prompt_token_ids, cache_salt=request.cache_salt)
# Add cache_salt if provided in the request
if request.cache_salt is not None:
engine_prompt["cache_salt"] = request.cache_salt
return messages, [engine_prompt]
return messages, [engine_input]
def create_error_response(
self,
@@ -450,7 +451,7 @@ class OpenAIServingRender:
request: Any,
prompt_input: str | list[str] | list[int] | list[list[int]] | None,
prompt_embeds: bytes | list[bytes] | None,
) -> list[ProcessorInputs]:
) -> list[EngineInput]:
"""Copied from OpenAIServing._preprocess_completion."""
prompts = list[SingletonPrompt | bytes]()
if prompt_embeds is not None: # embeds take higher priority
@@ -463,7 +464,7 @@ class OpenAIServingRender:
self,
request: Any,
prompts: Sequence[PromptType | bytes],
) -> list[ProcessorInputs]:
) -> list[EngineInput]:
"""Copied from OpenAIServing._preprocess_cmpl."""
renderer = self.renderer
model_config = self.model_config
@@ -497,7 +498,7 @@ class OpenAIServingRender:
default_template_kwargs: dict[str, Any] | None,
tool_dicts: list[dict[str, Any]] | None = None,
tool_parser: type[ToolParser] | None = None,
) -> tuple[list[ConversationMessage], list[ProcessorInputs]]:
) -> tuple[list[ConversationMessage], list[EngineInput]]:
"""Copied from OpenAIServing._preprocess_chat."""
renderer = self.renderer
mm_config = self.model_config.multimodal_config
@@ -519,7 +520,7 @@ class OpenAIServingRender:
default_mm_processor_kwargs=getattr(request, "mm_processor_kwargs", None),
)
(conversation,), (engine_prompt,) = await renderer.render_chat_async(
(conversation,), (engine_input,) = await renderer.render_chat_async(
[messages],
chat_params,
tok_params,
@@ -546,4 +547,4 @@ class OpenAIServingRender:
tokenizer = renderer.get_tokenizer()
request = tool_parser(tokenizer).adjust_request(request=request) # type: ignore[arg-type]
return conversation, [engine_prompt]
return conversation, [engine_input]