[Bugfix][Frontend] Validate reused prompt token ids; render messages for media and echo (#55771)

Signed-off-by: Shijie Lyu <sjlyu@amazon.com>
Co-authored-by: Shijie Lyu <sjlyu@amazon.com>
Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
Shijie Lyu
2026-09-29 10:36:22 +08:00
committed by GitHub
co-authored by Shijie Lyu Claude
parent 0af34418e9
commit c6843d397c
6 changed files with 239 additions and 2 deletions
+6 -1
View File
@@ -1810,6 +1810,11 @@ PART_TYPES_TO_SKIP_NONE_CONTENT = (
"refusal",
)
# Content part types parsed as text rather than multimodal data.
TEXT_PART_TYPES = frozenset(
{"text", "input_text", "output_text", "refusal", "thinking"}
)
def _parse_chat_message_content_parts(
role: str,
@@ -1904,7 +1909,7 @@ def _parse_chat_message_content_part(
)
return None
if part_type in ("text", "input_text", "output_text", "refusal", "thinking"):
if part_type in TEXT_PART_TYPES:
str_content = cast(str, content)
_reject_reserved_placeholder_in_text(str_content, mm_parser.model_config)
if wrap_dicts:
@@ -20,6 +20,8 @@ from pydantic import (
from vllm.config import ModelConfig
from vllm.entrypoints.chat_utils import (
MM_PARSER_MAP,
TEXT_PART_TYPES,
ChatCompletionMessageParam,
ChatTemplateContentFormatOption,
)
@@ -58,6 +60,11 @@ logger = init_logger(__name__)
_INT64_MIN = -(2**63)
_INT64_MAX = 2**63 - 1
# Content part types that carry no multimodal data.
_TEXT_CONTENT_PART_TYPES = TEXT_PART_TYPES | {"tool_reference"}
# Keys that mark a content part as multimodal, whatever its ``type``.
_MEDIA_CONTENT_PART_KEYS = frozenset(MM_PARSER_MAP) - _TEXT_CONTENT_PART_TYPES
class ChatMessage(OpenAIBaseModel):
role: str
@@ -976,6 +983,39 @@ class ChatCompletionRequest(OpenAIBaseModel):
)
return data
@model_validator(mode="before")
@classmethod
def drop_prompt_token_ids_with_media(cls, data):
# The forwarded ids would drop media in ``messages``, so ignore them. Runs
# before validation, which can turn content into a one-shot iterator.
if not isinstance(data, dict):
return data
kv_transfer_params = data.get("kv_transfer_params")
messages = data.get("messages")
if (
not isinstance(kv_transfer_params, dict)
or kv_transfer_params.get("prompt_token_ids") is None
or not isinstance(messages, list)
):
return data
for msg in messages:
content = msg.get("content") if isinstance(msg, dict) else None
if not isinstance(content, list):
continue
for part in content:
if isinstance(part, dict) and (
any(key in part for key in _MEDIA_CONTENT_PART_KEYS)
or not isinstance(part_type := part.get("type", "text"), str)
or part_type not in _TEXT_CONTENT_PART_TYPES
):
logger.debug(
"Ignoring kv_transfer_params['prompt_token_ids']: "
"messages have non-text content and are rendered instead."
)
kv_transfer_params.pop("prompt_token_ids")
return data
return data
@model_validator(mode="before")
@classmethod
def check_system_message_content_type(cls, data):
@@ -1117,6 +1157,17 @@ class BatchChatCompletionRequest(OpenAIBaseModel):
"when using `logprob_token_ids`, `logprobs` must be set to true.",
parameter="logprob_token_ids",
)
kv_transfer_params = data.get("kv_transfer_params")
if (
isinstance(kv_transfer_params, dict)
and kv_transfer_params.get("prompt_token_ids") is not None
):
raise VLLMValidationError(
"Batch chat completions do not support "
"`kv_transfer_params['prompt_token_ids']`: one pre-tokenized "
"prompt cannot serve several conversations.",
parameter="kv_transfer_params.prompt_token_ids",
)
response_format = data.get("response_format")
if response_format is not None:
rf_type = (