[Bugfix][Frontend] Validate reused prompt token ids; render messages for media and echo (#55771)
Signed-off-by: Shijie Lyu <sjlyu@amazon.com> Co-authored-by: Shijie Lyu <sjlyu@amazon.com> Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
co-authored by
Shijie Lyu
Claude
parent
0af34418e9
commit
c6843d397c
@@ -1810,6 +1810,11 @@ PART_TYPES_TO_SKIP_NONE_CONTENT = (
|
||||
"refusal",
|
||||
)
|
||||
|
||||
# Content part types parsed as text rather than multimodal data.
|
||||
TEXT_PART_TYPES = frozenset(
|
||||
{"text", "input_text", "output_text", "refusal", "thinking"}
|
||||
)
|
||||
|
||||
|
||||
def _parse_chat_message_content_parts(
|
||||
role: str,
|
||||
@@ -1904,7 +1909,7 @@ def _parse_chat_message_content_part(
|
||||
)
|
||||
return None
|
||||
|
||||
if part_type in ("text", "input_text", "output_text", "refusal", "thinking"):
|
||||
if part_type in TEXT_PART_TYPES:
|
||||
str_content = cast(str, content)
|
||||
_reject_reserved_placeholder_in_text(str_content, mm_parser.model_config)
|
||||
if wrap_dicts:
|
||||
|
||||
@@ -20,6 +20,8 @@ from pydantic import (
|
||||
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.entrypoints.chat_utils import (
|
||||
MM_PARSER_MAP,
|
||||
TEXT_PART_TYPES,
|
||||
ChatCompletionMessageParam,
|
||||
ChatTemplateContentFormatOption,
|
||||
)
|
||||
@@ -58,6 +60,11 @@ logger = init_logger(__name__)
|
||||
_INT64_MIN = -(2**63)
|
||||
_INT64_MAX = 2**63 - 1
|
||||
|
||||
# Content part types that carry no multimodal data.
|
||||
_TEXT_CONTENT_PART_TYPES = TEXT_PART_TYPES | {"tool_reference"}
|
||||
# Keys that mark a content part as multimodal, whatever its ``type``.
|
||||
_MEDIA_CONTENT_PART_KEYS = frozenset(MM_PARSER_MAP) - _TEXT_CONTENT_PART_TYPES
|
||||
|
||||
|
||||
class ChatMessage(OpenAIBaseModel):
|
||||
role: str
|
||||
@@ -976,6 +983,39 @@ class ChatCompletionRequest(OpenAIBaseModel):
|
||||
)
|
||||
return data
|
||||
|
||||
@model_validator(mode="before")
|
||||
@classmethod
|
||||
def drop_prompt_token_ids_with_media(cls, data):
|
||||
# The forwarded ids would drop media in ``messages``, so ignore them. Runs
|
||||
# before validation, which can turn content into a one-shot iterator.
|
||||
if not isinstance(data, dict):
|
||||
return data
|
||||
kv_transfer_params = data.get("kv_transfer_params")
|
||||
messages = data.get("messages")
|
||||
if (
|
||||
not isinstance(kv_transfer_params, dict)
|
||||
or kv_transfer_params.get("prompt_token_ids") is None
|
||||
or not isinstance(messages, list)
|
||||
):
|
||||
return data
|
||||
for msg in messages:
|
||||
content = msg.get("content") if isinstance(msg, dict) else None
|
||||
if not isinstance(content, list):
|
||||
continue
|
||||
for part in content:
|
||||
if isinstance(part, dict) and (
|
||||
any(key in part for key in _MEDIA_CONTENT_PART_KEYS)
|
||||
or not isinstance(part_type := part.get("type", "text"), str)
|
||||
or part_type not in _TEXT_CONTENT_PART_TYPES
|
||||
):
|
||||
logger.debug(
|
||||
"Ignoring kv_transfer_params['prompt_token_ids']: "
|
||||
"messages have non-text content and are rendered instead."
|
||||
)
|
||||
kv_transfer_params.pop("prompt_token_ids")
|
||||
return data
|
||||
return data
|
||||
|
||||
@model_validator(mode="before")
|
||||
@classmethod
|
||||
def check_system_message_content_type(cls, data):
|
||||
@@ -1117,6 +1157,17 @@ class BatchChatCompletionRequest(OpenAIBaseModel):
|
||||
"when using `logprob_token_ids`, `logprobs` must be set to true.",
|
||||
parameter="logprob_token_ids",
|
||||
)
|
||||
kv_transfer_params = data.get("kv_transfer_params")
|
||||
if (
|
||||
isinstance(kv_transfer_params, dict)
|
||||
and kv_transfer_params.get("prompt_token_ids") is not None
|
||||
):
|
||||
raise VLLMValidationError(
|
||||
"Batch chat completions do not support "
|
||||
"`kv_transfer_params['prompt_token_ids']`: one pre-tokenized "
|
||||
"prompt cannot serve several conversations.",
|
||||
parameter="kv_transfer_params.prompt_token_ids",
|
||||
)
|
||||
response_format = data.get("response_format")
|
||||
if response_format is not None:
|
||||
rf_type = (
|
||||
|
||||
Reference in New Issue
Block a user