diff --git a/tests/entrypoints/openai/responses/test_serving_responses.py b/tests/entrypoints/openai/responses/test_serving_responses.py index d07a89ec9c..591ba5c42a 100644 --- a/tests/entrypoints/openai/responses/test_serving_responses.py +++ b/tests/entrypoints/openai/responses/test_serving_responses.py @@ -34,6 +34,7 @@ from openai_harmony import Message as OpenAIHarmonyMessage from openai_harmony import Role import vllm.envs as envs +from tests.entrypoints.openai.chat_completion.test_serving_chat import MockModelConfig from vllm.entrypoints.generate.base.protocol import ( DeltaFunctionCall, DeltaMessage, @@ -294,6 +295,102 @@ async def test_online_renderer_renders_non_harmony_responses_with_explicit_histo assert preprocess_kwargs["default_template_kwargs"]["request_default"] == "kept" +_REUSED_IDS = [10, 20, 30] +_IMAGE_PART = { + "type": "input_image", + "image_url": "https://example.com/a.png", + "detail": "auto", +} + + +async def _render_responses_with_reuse(request, previous_messages=None): + online_renderer = OnlineRenderer( + model_config=MockModelConfig(), + renderer=MagicMock(), + request_logger=None, + chat_template=None, + chat_template_content_format="auto", + ) + render_chat = AsyncMock(return_value=([[]], [tokens_input([1, 2])])) + online_renderer.renderer.render_chat_async = render_chat + + result = await online_renderer.render_responses( + request, previous_messages=previous_messages + ) + + assert not isinstance(result, ErrorResponse) + return render_chat, result.engine_input + + +@pytest.mark.asyncio +@pytest.mark.parametrize("ids", [_REUSED_IDS, [1.5]]) +@pytest.mark.parametrize( + "request_input", + [ + [ + { + "role": "user", + "content": [{"type": "input_text", "text": "look"}, _IMAGE_PART], + } + ], + [ + { + "type": "function_call", + "call_id": "call_1", + "name": "shot", + "arguments": "{}", + }, + { + "type": "function_call_output", + "call_id": "call_1", + "output": [_IMAGE_PART], + }, + ], + ], + ids=["input_image", "tool_output_image"], +) +async def test_responses_kv_transfer_prompt_token_ids_ignored_with_non_text_input( + request_input, ids +): + request = ResponsesRequest( + input=request_input, + kv_transfer_params={"do_remote_prefill": True, "prompt_token_ids": ids}, + ) + + render_chat, _ = await _render_responses_with_reuse(request) + + render_chat.assert_awaited_once() + assert request.kv_transfer_params == {"do_remote_prefill": True} + + +@pytest.mark.asyncio +async def test_responses_kv_transfer_prompt_token_ids_ignored_with_media_in_history(): + request = ResponsesRequest( + input="next", kv_transfer_params={"prompt_token_ids": _REUSED_IDS} + ) + + render_chat, _ = await _render_responses_with_reuse( + request, previous_messages=[{"role": "user", "content": [_IMAGE_PART]}] + ) + + render_chat.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_responses_kv_transfer_prompt_token_ids_kept_with_text_input(): + request = ResponsesRequest( + input=[ + {"role": "user", "content": [{"type": "input_text", "text": "hi"}]}, + {"role": "assistant", "content": "ok"}, + ], + kv_transfer_params={"prompt_token_ids": _REUSED_IDS}, + ) + + _, engine_input = await _render_responses_with_reuse(request) + + assert engine_input["prompt_token_ids"] == _REUSED_IDS + + @pytest.mark.asyncio async def test_online_renderer_rejects_untrusted_responses_chat_template(): renderer = _new_online_renderer() diff --git a/vllm/entrypoints/chat_utils.py b/vllm/entrypoints/chat_utils.py index a419ffca81..10c4866f89 100644 --- a/vllm/entrypoints/chat_utils.py +++ b/vllm/entrypoints/chat_utils.py @@ -1814,6 +1814,32 @@ PART_TYPES_TO_SKIP_NONE_CONTENT = ( TEXT_PART_TYPES = frozenset( {"text", "input_text", "output_text", "refusal", "thinking"} ) +# Content part types that carry no multimodal data. +_TEXT_CONTENT_PART_TYPES = TEXT_PART_TYPES | {"tool_reference"} +# Keys that mark a content part as multimodal, whatever its ``type``. +_MEDIA_CONTENT_PART_KEYS = frozenset(MM_PARSER_MAP) - _TEXT_CONTENT_PART_TYPES + + +def has_non_text_content(messages: Any) -> bool: + """Whether any message in ``messages`` has a non-text content part. + + Only list content is inspected, so validated chat content that is a + one-shot iterator is never consumed. + """ + if not isinstance(messages, list): + return False + for msg in messages: + content = msg.get("content") if isinstance(msg, dict) else None + if not isinstance(content, list): + continue + for part in content: + if isinstance(part, dict) and ( + any(key in part for key in _MEDIA_CONTENT_PART_KEYS) + or not isinstance(part_type := part.get("type", "text"), str) + or part_type not in _TEXT_CONTENT_PART_TYPES + ): + return True + return False def _parse_chat_message_content_parts( diff --git a/vllm/entrypoints/openai/chat_completion/protocol.py b/vllm/entrypoints/openai/chat_completion/protocol.py index 8ad7534d6d..228472ab6b 100644 --- a/vllm/entrypoints/openai/chat_completion/protocol.py +++ b/vllm/entrypoints/openai/chat_completion/protocol.py @@ -20,10 +20,9 @@ from pydantic import ( from vllm.config import ModelConfig from vllm.entrypoints.chat_utils import ( - MM_PARSER_MAP, - TEXT_PART_TYPES, ChatCompletionMessageParam, ChatTemplateContentFormatOption, + has_non_text_content, ) from vllm.entrypoints.generate.base.protocol import ( AnyResponseFormat, @@ -60,11 +59,6 @@ logger = init_logger(__name__) _INT64_MIN = -(2**63) _INT64_MAX = 2**63 - 1 -# Content part types that carry no multimodal data. -_TEXT_CONTENT_PART_TYPES = TEXT_PART_TYPES | {"tool_reference"} -# Keys that mark a content part as multimodal, whatever its ``type``. -_MEDIA_CONTENT_PART_KEYS = frozenset(MM_PARSER_MAP) - _TEXT_CONTENT_PART_TYPES - class ChatMessage(OpenAIBaseModel): role: str @@ -991,29 +985,16 @@ class ChatCompletionRequest(OpenAIBaseModel): if not isinstance(data, dict): return data kv_transfer_params = data.get("kv_transfer_params") - messages = data.get("messages") if ( - not isinstance(kv_transfer_params, dict) - or kv_transfer_params.get("prompt_token_ids") is None - or not isinstance(messages, list) + isinstance(kv_transfer_params, dict) + and kv_transfer_params.get("prompt_token_ids") is not None + and has_non_text_content(data.get("messages")) ): - return data - for msg in messages: - content = msg.get("content") if isinstance(msg, dict) else None - if not isinstance(content, list): - continue - for part in content: - if isinstance(part, dict) and ( - any(key in part for key in _MEDIA_CONTENT_PART_KEYS) - or not isinstance(part_type := part.get("type", "text"), str) - or part_type not in _TEXT_CONTENT_PART_TYPES - ): - logger.debug( - "Ignoring kv_transfer_params['prompt_token_ids']: " - "messages have non-text content and are rendered instead." - ) - kv_transfer_params.pop("prompt_token_ids") - return data + logger.debug( + "Ignoring kv_transfer_params['prompt_token_ids']: " + "messages have non-text content and are rendered instead." + ) + kv_transfer_params.pop("prompt_token_ids") return data @model_validator(mode="before") diff --git a/vllm/renderers/online_renderer.py b/vllm/renderers/online_renderer.py index 5108adfe81..bbe24f7df3 100644 --- a/vllm/renderers/online_renderer.py +++ b/vllm/renderers/online_renderer.py @@ -16,6 +16,7 @@ from vllm.entrypoints.chat_utils import ( ChatCompletionMessageParam, ChatTemplateContentFormatOption, ConversationMessage, + has_non_text_content, ) from vllm.entrypoints.generate.base.protocol import validate_request_mm_kwargs from vllm.entrypoints.openai.chat_completion.protocol import ( @@ -101,16 +102,18 @@ def _extract_allowed_tools_from_mcp_requests( return allowed_tools_map -def _reused_prompt_token_ids(request: Any) -> list[int] | None: +def _reused_prompt_token_ids( + request: Any, messages: list[Any] | None = None +) -> list[int] | None: """Pop prompt token ids forwarded for decode-side reuse, if any. Disaggregated serving carries the prefill stage's ids in ``kv_transfer_params`` so the decode stage can skip re-tokenizing. Removing the key keeps the id list out of the engine's sampling metadata. - Returns None without checking the ids when ``echo`` is set, since echo - needs ``messages`` to be rendered. Otherwise raises VLLMValidationError if - the ids are malformed. + Returns None without checking the ids when ``echo`` is set or ``messages`` + has non-text content, since both need ``messages`` to be rendered. + Otherwise raises VLLMValidationError if the ids are malformed. """ kv = getattr(request, "kv_transfer_params", None) if not isinstance(kv, dict): @@ -124,6 +127,12 @@ def _reused_prompt_token_ids(request: Any) -> list[int] | None: "echo is set, so messages are rendered instead." ) return None + if has_non_text_content(messages): + logger.debug( + "Ignoring kv_transfer_params['prompt_token_ids']: " + "messages have non-text content and are rendered instead." + ) + return None # bool is an int subclass, hence the exact type check. if ( not isinstance(ids, list) @@ -772,7 +781,7 @@ class OnlineRenderer: default_mm_processor_kwargs=getattr(request, "mm_processor_kwargs", None), ) - reuse_ids = _reused_prompt_token_ids(request) + reuse_ids = _reused_prompt_token_ids(request, messages) if reuse_ids: # Decode-side token reuse: feed the forwarded ids straight to the # engine, skipping templating and tokenization. ``messages`` are not