[Bug] Fix Failure in /v1/chat/completions/render for Multimodal Requests (https://github.com/vllm-project/vllm/issues/35665) (#35684)

This commit is contained in:
Sergey Zinchenko
2026-03-14 14:10:11 +00:00
committed by GitHub
parent 600a039f57
commit 4a718e770d
13 changed files with 560 additions and 168 deletions
+146 -6
View File
@@ -24,14 +24,29 @@ from vllm.entrypoints.openai.parser.harmony_utils import (
parse_chat_inputs_to_harmony_messages,
render_for_completion,
)
from vllm.entrypoints.utils import create_error_response
from vllm.entrypoints.serve.disagg.protocol import (
GenerateRequest,
MultiModalFeatures,
PlaceholderRangeInfo,
)
from vllm.entrypoints.utils import (
create_error_response,
get_max_tokens,
)
from vllm.inputs.data import ProcessorInputs, PromptType, SingletonPrompt, TokensPrompt
from vllm.logger import init_logger
from vllm.multimodal.inputs import MultiModalHashes, MultiModalPlaceholderDict
from vllm.parser import ParserManager
from vllm.renderers import BaseRenderer, merge_kwargs
from vllm.renderers.inputs.preprocess import parse_model_prompt, prompt_to_seq
from vllm.renderers.inputs.preprocess import (
extract_prompt_components,
extract_prompt_len,
parse_model_prompt,
prompt_to_seq,
)
from vllm.tokenizers import TokenizerLike
from vllm.tool_parsers import ToolParser
from vllm.utils import random_uuid
from vllm.utils.mistral import is_mistral_tokenizer
from vllm.utils.mistral import mt as _mt
@@ -83,10 +98,18 @@ class OpenAIServingRender:
self.supports_browsing = False
self.supports_code_interpreter = False
self.default_sampling_params = model_config.get_diff_sampling_param()
mc = model_config
self.override_max_tokens = (
self.default_sampling_params.get("max_tokens")
if mc.generation_config not in ("auto", "vllm")
else getattr(mc, "override_generation_config", {}).get("max_new_tokens")
)
async def render_chat_request(
self,
request: ChatCompletionRequest,
) -> tuple[list[ConversationMessage], list[ProcessorInputs]] | ErrorResponse:
) -> GenerateRequest | ErrorResponse:
"""Validate the model and preprocess a chat completion request.
This is the authoritative implementation used directly by the
@@ -96,7 +119,56 @@ class OpenAIServingRender:
if error_check_ret is not None:
logger.error("Error with model %s", error_check_ret)
return error_check_ret
return await self.render_chat(request)
if request.use_beam_search:
return self.create_error_response(
"Beam search is not supported by the render endpoint"
)
result = await self.render_chat(request)
if isinstance(result, ErrorResponse):
return result
_, engine_prompts = result
if len(engine_prompts) != 1:
return self.create_error_response(
f"Expected exactly 1 engine prompt, got {len(engine_prompts)}"
)
engine_prompt = engine_prompts[0]
prompt_components = extract_prompt_components(self.model_config, engine_prompt)
token_ids = prompt_components.token_ids
if not token_ids:
return self.create_error_response("No token_ids rendered")
token_ids = list(token_ids)
input_length = extract_prompt_len(self.model_config, engine_prompt)
max_tokens = get_max_tokens(
self.model_config.max_model_len,
request.max_completion_tokens
if request.max_completion_tokens is not None
else request.max_tokens,
input_length,
self.default_sampling_params,
self.override_max_tokens,
)
params = request.to_sampling_params(max_tokens, self.default_sampling_params)
request_id = f"chatcmpl-{random_uuid()}"
return GenerateRequest(
request_id=request_id,
token_ids=token_ids,
features=self._extract_mm_features(engine_prompt),
sampling_params=params,
model=request.model,
stream=bool(request.stream),
stream_options=(request.stream_options if request.stream else None),
cache_salt=request.cache_salt,
priority=request.priority,
)
async def render_chat(
self,
@@ -183,7 +255,7 @@ class OpenAIServingRender:
async def render_completion_request(
self,
request: CompletionRequest,
) -> list[ProcessorInputs] | ErrorResponse:
) -> list[GenerateRequest] | ErrorResponse:
"""Validate the model and preprocess a completion request.
This is the authoritative implementation used directly by the
@@ -192,7 +264,48 @@ class OpenAIServingRender:
error_check_ret = await self._check_model(request)
if error_check_ret is not None:
return error_check_ret
return await self.render_completion(request)
result = await self.render_completion(request)
if isinstance(result, ErrorResponse):
return result
generate_requests: list[GenerateRequest] = []
for engine_prompt in result:
prompt_components = extract_prompt_components(
self.model_config, engine_prompt
)
token_ids = prompt_components.token_ids
if not token_ids:
return self.create_error_response("No token_ids rendered")
token_ids = list(token_ids)
input_length = extract_prompt_len(self.model_config, engine_prompt)
max_tokens = get_max_tokens(
self.model_config.max_model_len,
request.max_tokens,
input_length,
self.default_sampling_params,
self.override_max_tokens,
)
params = request.to_sampling_params(
max_tokens, self.default_sampling_params
)
request_id = f"cmpl-{random_uuid()}"
generate_requests.append(
GenerateRequest(
request_id=request_id,
token_ids=token_ids,
features=self._extract_mm_features(engine_prompt),
sampling_params=params,
model=request.model,
stream=bool(request.stream),
stream_options=(request.stream_options if request.stream else None),
cache_salt=request.cache_salt,
priority=request.priority,
)
)
return generate_requests
async def render_completion(
self,
@@ -223,6 +336,33 @@ class OpenAIServingRender:
return engine_prompts
@staticmethod
def _extract_mm_features(
engine_prompt: ProcessorInputs,
) -> MultiModalFeatures | None:
"""Extract multimodal metadata from a rendered engine prompt.
Returns ``None`` for text-only prompts.
"""
if engine_prompt.get("type") != "multimodal":
return None
# At this point engine_prompt is a MultiModalInputs TypedDict.
mm_hashes: MultiModalHashes = engine_prompt["mm_hashes"] # type: ignore[typeddict-item]
raw_placeholders: MultiModalPlaceholderDict = engine_prompt["mm_placeholders"] # type: ignore[typeddict-item]
mm_placeholders = {
modality: [
PlaceholderRangeInfo(offset=p.offset, length=p.length) for p in ranges
]
for modality, ranges in raw_placeholders.items()
}
return MultiModalFeatures(
mm_hashes=mm_hashes,
mm_placeholders=mm_placeholders,
)
def _make_request_with_harmony(
self,
request: ChatCompletionRequest,