[Bugfix][Frontend] Ignore reused prompt token ids for media in /v1/responses (#59173)
Signed-off-by: Shijie Lyu <lshjhf@gmail.com>
This commit is contained in:
@@ -34,6 +34,7 @@ from openai_harmony import Message as OpenAIHarmonyMessage
|
||||
from openai_harmony import Role
|
||||
|
||||
import vllm.envs as envs
|
||||
from tests.entrypoints.openai.chat_completion.test_serving_chat import MockModelConfig
|
||||
from vllm.entrypoints.generate.base.protocol import (
|
||||
DeltaFunctionCall,
|
||||
DeltaMessage,
|
||||
@@ -294,6 +295,102 @@ async def test_online_renderer_renders_non_harmony_responses_with_explicit_histo
|
||||
assert preprocess_kwargs["default_template_kwargs"]["request_default"] == "kept"
|
||||
|
||||
|
||||
_REUSED_IDS = [10, 20, 30]
|
||||
_IMAGE_PART = {
|
||||
"type": "input_image",
|
||||
"image_url": "https://example.com/a.png",
|
||||
"detail": "auto",
|
||||
}
|
||||
|
||||
|
||||
async def _render_responses_with_reuse(request, previous_messages=None):
|
||||
online_renderer = OnlineRenderer(
|
||||
model_config=MockModelConfig(),
|
||||
renderer=MagicMock(),
|
||||
request_logger=None,
|
||||
chat_template=None,
|
||||
chat_template_content_format="auto",
|
||||
)
|
||||
render_chat = AsyncMock(return_value=([[]], [tokens_input([1, 2])]))
|
||||
online_renderer.renderer.render_chat_async = render_chat
|
||||
|
||||
result = await online_renderer.render_responses(
|
||||
request, previous_messages=previous_messages
|
||||
)
|
||||
|
||||
assert not isinstance(result, ErrorResponse)
|
||||
return render_chat, result.engine_input
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("ids", [_REUSED_IDS, [1.5]])
|
||||
@pytest.mark.parametrize(
|
||||
"request_input",
|
||||
[
|
||||
[
|
||||
{
|
||||
"role": "user",
|
||||
"content": [{"type": "input_text", "text": "look"}, _IMAGE_PART],
|
||||
}
|
||||
],
|
||||
[
|
||||
{
|
||||
"type": "function_call",
|
||||
"call_id": "call_1",
|
||||
"name": "shot",
|
||||
"arguments": "{}",
|
||||
},
|
||||
{
|
||||
"type": "function_call_output",
|
||||
"call_id": "call_1",
|
||||
"output": [_IMAGE_PART],
|
||||
},
|
||||
],
|
||||
],
|
||||
ids=["input_image", "tool_output_image"],
|
||||
)
|
||||
async def test_responses_kv_transfer_prompt_token_ids_ignored_with_non_text_input(
|
||||
request_input, ids
|
||||
):
|
||||
request = ResponsesRequest(
|
||||
input=request_input,
|
||||
kv_transfer_params={"do_remote_prefill": True, "prompt_token_ids": ids},
|
||||
)
|
||||
|
||||
render_chat, _ = await _render_responses_with_reuse(request)
|
||||
|
||||
render_chat.assert_awaited_once()
|
||||
assert request.kv_transfer_params == {"do_remote_prefill": True}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_responses_kv_transfer_prompt_token_ids_ignored_with_media_in_history():
|
||||
request = ResponsesRequest(
|
||||
input="next", kv_transfer_params={"prompt_token_ids": _REUSED_IDS}
|
||||
)
|
||||
|
||||
render_chat, _ = await _render_responses_with_reuse(
|
||||
request, previous_messages=[{"role": "user", "content": [_IMAGE_PART]}]
|
||||
)
|
||||
|
||||
render_chat.assert_awaited_once()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_responses_kv_transfer_prompt_token_ids_kept_with_text_input():
|
||||
request = ResponsesRequest(
|
||||
input=[
|
||||
{"role": "user", "content": [{"type": "input_text", "text": "hi"}]},
|
||||
{"role": "assistant", "content": "ok"},
|
||||
],
|
||||
kv_transfer_params={"prompt_token_ids": _REUSED_IDS},
|
||||
)
|
||||
|
||||
_, engine_input = await _render_responses_with_reuse(request)
|
||||
|
||||
assert engine_input["prompt_token_ids"] == _REUSED_IDS
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_online_renderer_rejects_untrusted_responses_chat_template():
|
||||
renderer = _new_online_renderer()
|
||||
|
||||
@@ -1814,6 +1814,32 @@ PART_TYPES_TO_SKIP_NONE_CONTENT = (
|
||||
TEXT_PART_TYPES = frozenset(
|
||||
{"text", "input_text", "output_text", "refusal", "thinking"}
|
||||
)
|
||||
# Content part types that carry no multimodal data.
|
||||
_TEXT_CONTENT_PART_TYPES = TEXT_PART_TYPES | {"tool_reference"}
|
||||
# Keys that mark a content part as multimodal, whatever its ``type``.
|
||||
_MEDIA_CONTENT_PART_KEYS = frozenset(MM_PARSER_MAP) - _TEXT_CONTENT_PART_TYPES
|
||||
|
||||
|
||||
def has_non_text_content(messages: Any) -> bool:
|
||||
"""Whether any message in ``messages`` has a non-text content part.
|
||||
|
||||
Only list content is inspected, so validated chat content that is a
|
||||
one-shot iterator is never consumed.
|
||||
"""
|
||||
if not isinstance(messages, list):
|
||||
return False
|
||||
for msg in messages:
|
||||
content = msg.get("content") if isinstance(msg, dict) else None
|
||||
if not isinstance(content, list):
|
||||
continue
|
||||
for part in content:
|
||||
if isinstance(part, dict) and (
|
||||
any(key in part for key in _MEDIA_CONTENT_PART_KEYS)
|
||||
or not isinstance(part_type := part.get("type", "text"), str)
|
||||
or part_type not in _TEXT_CONTENT_PART_TYPES
|
||||
):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _parse_chat_message_content_parts(
|
||||
|
||||
@@ -20,10 +20,9 @@ from pydantic import (
|
||||
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.entrypoints.chat_utils import (
|
||||
MM_PARSER_MAP,
|
||||
TEXT_PART_TYPES,
|
||||
ChatCompletionMessageParam,
|
||||
ChatTemplateContentFormatOption,
|
||||
has_non_text_content,
|
||||
)
|
||||
from vllm.entrypoints.generate.base.protocol import (
|
||||
AnyResponseFormat,
|
||||
@@ -60,11 +59,6 @@ logger = init_logger(__name__)
|
||||
_INT64_MIN = -(2**63)
|
||||
_INT64_MAX = 2**63 - 1
|
||||
|
||||
# Content part types that carry no multimodal data.
|
||||
_TEXT_CONTENT_PART_TYPES = TEXT_PART_TYPES | {"tool_reference"}
|
||||
# Keys that mark a content part as multimodal, whatever its ``type``.
|
||||
_MEDIA_CONTENT_PART_KEYS = frozenset(MM_PARSER_MAP) - _TEXT_CONTENT_PART_TYPES
|
||||
|
||||
|
||||
class ChatMessage(OpenAIBaseModel):
|
||||
role: str
|
||||
@@ -991,29 +985,16 @@ class ChatCompletionRequest(OpenAIBaseModel):
|
||||
if not isinstance(data, dict):
|
||||
return data
|
||||
kv_transfer_params = data.get("kv_transfer_params")
|
||||
messages = data.get("messages")
|
||||
if (
|
||||
not isinstance(kv_transfer_params, dict)
|
||||
or kv_transfer_params.get("prompt_token_ids") is None
|
||||
or not isinstance(messages, list)
|
||||
isinstance(kv_transfer_params, dict)
|
||||
and kv_transfer_params.get("prompt_token_ids") is not None
|
||||
and has_non_text_content(data.get("messages"))
|
||||
):
|
||||
return data
|
||||
for msg in messages:
|
||||
content = msg.get("content") if isinstance(msg, dict) else None
|
||||
if not isinstance(content, list):
|
||||
continue
|
||||
for part in content:
|
||||
if isinstance(part, dict) and (
|
||||
any(key in part for key in _MEDIA_CONTENT_PART_KEYS)
|
||||
or not isinstance(part_type := part.get("type", "text"), str)
|
||||
or part_type not in _TEXT_CONTENT_PART_TYPES
|
||||
):
|
||||
logger.debug(
|
||||
"Ignoring kv_transfer_params['prompt_token_ids']: "
|
||||
"messages have non-text content and are rendered instead."
|
||||
)
|
||||
kv_transfer_params.pop("prompt_token_ids")
|
||||
return data
|
||||
logger.debug(
|
||||
"Ignoring kv_transfer_params['prompt_token_ids']: "
|
||||
"messages have non-text content and are rendered instead."
|
||||
)
|
||||
kv_transfer_params.pop("prompt_token_ids")
|
||||
return data
|
||||
|
||||
@model_validator(mode="before")
|
||||
|
||||
@@ -16,6 +16,7 @@ from vllm.entrypoints.chat_utils import (
|
||||
ChatCompletionMessageParam,
|
||||
ChatTemplateContentFormatOption,
|
||||
ConversationMessage,
|
||||
has_non_text_content,
|
||||
)
|
||||
from vllm.entrypoints.generate.base.protocol import validate_request_mm_kwargs
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
@@ -101,16 +102,18 @@ def _extract_allowed_tools_from_mcp_requests(
|
||||
return allowed_tools_map
|
||||
|
||||
|
||||
def _reused_prompt_token_ids(request: Any) -> list[int] | None:
|
||||
def _reused_prompt_token_ids(
|
||||
request: Any, messages: list[Any] | None = None
|
||||
) -> list[int] | None:
|
||||
"""Pop prompt token ids forwarded for decode-side reuse, if any.
|
||||
|
||||
Disaggregated serving carries the prefill stage's ids in
|
||||
``kv_transfer_params`` so the decode stage can skip re-tokenizing. Removing
|
||||
the key keeps the id list out of the engine's sampling metadata.
|
||||
|
||||
Returns None without checking the ids when ``echo`` is set, since echo
|
||||
needs ``messages`` to be rendered. Otherwise raises VLLMValidationError if
|
||||
the ids are malformed.
|
||||
Returns None without checking the ids when ``echo`` is set or ``messages``
|
||||
has non-text content, since both need ``messages`` to be rendered.
|
||||
Otherwise raises VLLMValidationError if the ids are malformed.
|
||||
"""
|
||||
kv = getattr(request, "kv_transfer_params", None)
|
||||
if not isinstance(kv, dict):
|
||||
@@ -124,6 +127,12 @@ def _reused_prompt_token_ids(request: Any) -> list[int] | None:
|
||||
"echo is set, so messages are rendered instead."
|
||||
)
|
||||
return None
|
||||
if has_non_text_content(messages):
|
||||
logger.debug(
|
||||
"Ignoring kv_transfer_params['prompt_token_ids']: "
|
||||
"messages have non-text content and are rendered instead."
|
||||
)
|
||||
return None
|
||||
# bool is an int subclass, hence the exact type check.
|
||||
if (
|
||||
not isinstance(ids, list)
|
||||
@@ -772,7 +781,7 @@ class OnlineRenderer:
|
||||
default_mm_processor_kwargs=getattr(request, "mm_processor_kwargs", None),
|
||||
)
|
||||
|
||||
reuse_ids = _reused_prompt_token_ids(request)
|
||||
reuse_ids = _reused_prompt_token_ids(request, messages)
|
||||
if reuse_ids:
|
||||
# Decode-side token reuse: feed the forwarded ids straight to the
|
||||
# engine, skipping templating and tokenization. ``messages`` are not
|
||||
|
||||
Reference in New Issue
Block a user