diff --git a/tests/entrypoints/openai/chat_completion/test_extra_content_fields.py b/tests/entrypoints/openai/chat_completion/test_extra_content_fields.py new file mode 100644 index 0000000000..8c03fc1daa --- /dev/null +++ b/tests/entrypoints/openai/chat_completion/test_extra_content_fields.py @@ -0,0 +1,135 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""End-to-end tests for the TranslateGemma extra-fields-on-content-parts pathway. + +TranslateGemma's bundled chat template reads ``source_lang_code`` and +``target_lang_code`` off each content part to assemble the translation prompt. +This PR makes sure those extra fields survive request parsing and reach the +template; these tests verify the full pathway with the real model on both +the text and image content branches of the template. +""" + +import json + +import openai +import pytest +import pytest_asyncio + +from tests.utils import RemoteOpenAIServer +from vllm.assets.image import ImageAsset +from vllm.multimodal.utils import encode_image_url + +MODEL_NAME = "google/translategemma-4b-it" + + +@pytest.fixture(scope="module") +def server(): + args = [ + "--max-model-len", + "2048", + "--max-num-seqs", + "16", + "--enforce-eager", + "--chat-template-content-format", + "openai", + "--limit-mm-per-prompt", + json.dumps({"image": 1}), + ] + with RemoteOpenAIServer(MODEL_NAME, args) as remote_server: + yield remote_server + + +@pytest_asyncio.fixture +async def client(server): + async with server.get_async_client() as async_client: + yield async_client + + +@pytest.fixture(scope="module") +def stop_sign_image_url(): + return encode_image_url(ImageAsset("stop_sign").pil_image) + + +@pytest.mark.asyncio +async def test_translategemma_extra_lang_code_fields(client: openai.AsyncOpenAI): + """en -> es translation through TranslateGemma's bundled chat template's + ``text`` content branch, which depends on ``source_lang_code`` / + ``target_lang_code`` extra fields being preserved on the content part.""" + completion = await client.chat.completions.create( + model=MODEL_NAME, + messages=[ + { + "role": "user", + "content": [ + { + "type": "text", + "text": "The quick brown fox jumps over the lazy dog.", + "source_lang_code": "en", + "target_lang_code": "es", + } + ], + } + ], + max_tokens=64, + temperature=0.0, + ) + + content = completion.choices[0].message.content + assert content is not None and content.strip(), ( + "expected a non-empty Spanish translation" + ) + # If extra fields had been stripped, the bundled template would not + # produce a translation prompt and the output would not be Spanish. + spanish_markers = (" el ", " la ", " los ", " las ", " perro", " zorro") + lowered = f" {content.lower()} " + assert any(m in lowered for m in spanish_markers), ( + f"output does not look like Spanish: {content!r}" + ) + + +@pytest.mark.asyncio +async def test_translategemma_image_extra_lang_code_fields( + client: openai.AsyncOpenAI, + stop_sign_image_url: str, +): + """en -> es OCR-translation through TranslateGemma's bundled chat + template's ``image`` content branch. Exercises the multimodal branch of + ``_collect_extra_fields`` in ``_parse_chat_message_content_part``: the + ``source_lang_code`` / ``target_lang_code`` extras must survive parsing + and end up alongside the ``{"type": "image"}`` placeholder so the + template's image branch (``content["type"] == 'image'``) renders the + right OCR-translation prompt.""" + completion = await client.chat.completions.create( + model=MODEL_NAME, + messages=[ + { + "role": "user", + "content": [ + { + "type": "image_url", + "image_url": {"url": stop_sign_image_url}, + "source_lang_code": "en", + "target_lang_code": "es", + } + ], + } + ], + max_tokens=64, + temperature=0.0, + ) + + content = completion.choices[0].message.content + assert content is not None and content.strip(), ( + "expected a non-empty translation of the stop-sign image" + ) + # Common Spanish renderings of "STOP" on a road sign. If extras were + # stripped we would either fail the template or get back the original + # English word, neither of which match this set. + spanish_stop_terms = ("detén", "deteng", "deten", "alto", "pare", "para") + lowered = content.lower() + assert "stop" not in lowered, ( + f"output appears to echo the English source: {content!r}" + ) + assert any(t in lowered for t in spanish_stop_terms), ( + f"output is not a recognizable Spanish translation of 'STOP': {content!r}" + ) diff --git a/vllm/entrypoints/chat_utils.py b/vllm/entrypoints/chat_utils.py index b7d96c894a..c89e9fa79d 100644 --- a/vllm/entrypoints/chat_utils.py +++ b/vllm/entrypoints/chat_utils.py @@ -3,6 +3,7 @@ import asyncio import json +import types from abc import ABC, abstractmethod from collections import Counter, defaultdict from collections.abc import Awaitable, Callable, Iterable @@ -10,7 +11,19 @@ from dataclasses import dataclass from functools import cached_property, lru_cache, partial from itertools import accumulate from pathlib import Path -from typing import TYPE_CHECKING, Any, Final, Generic, Literal, TypeAlias, TypeVar, cast +from typing import ( + TYPE_CHECKING, + Any, + Final, + Generic, + Literal, + TypeAlias, + TypeVar, + Union, + cast, + get_args, + get_origin, +) from openai.types.chat import ( ChatCompletionAssistantMessageParam, @@ -1460,6 +1473,25 @@ MM_PARSER_MAP: dict[ } +def _collect_known_content_part_fields() -> frozenset[str]: + fields: set[str] = set() + stack: list[Any] = [ChatCompletionContentPartParam] + while stack: + node = stack.pop() + if get_origin(node) in (Union, types.UnionType): + stack.extend(get_args(node)) + elif hasattr(node, "__required_keys__"): + fields |= node.__required_keys__ | node.__optional_keys__ + return frozenset(fields) + + +_KNOWN_CONTENT_PART_FIELDS = _collect_known_content_part_fields() + + +def _collect_extra_fields(part: dict[str, Any]) -> dict[str, Any]: + return {k: v for k, v in part.items() if k not in _KNOWN_CONTENT_PART_FIELDS} + + def _parse_chat_message_content_mm_part( part: ChatCompletionContentPartParam, ) -> tuple[str, _ContentPart]: @@ -1669,7 +1701,9 @@ def _parse_chat_message_content_part( str_content = cast(str, content) _reject_reserved_placeholder_in_text(str_content, mm_parser.model_config) if wrap_dicts: - return {"type": "text", "text": str_content} + result: dict[str, Any] = {"type": "text", "text": str_content} + result.update(_collect_extra_fields(cast(dict[str, Any], part))) + return result else: return str_content @@ -1734,7 +1768,9 @@ def _parse_chat_message_content_part( # emit the single sentinel token as text so the template renders # it inline. The renderer later expands it to N tokens post-tokenize. return {"type": "text", "text": PROMPT_EMBEDS_PLACEHOLDER_TOKEN} - return {"type": modality} + result = {"type": modality} + result.update(_collect_extra_fields(cast(dict[str, Any], part))) + return result if modality == "prompt_embeds": # Emit the renderer token inline regardless of `interleave_strings`, # prompt_embeds are spliced at the token offset so position matters.