[Misc] Break chat_utils into more maintainable modules. (#60972)

Signed-off-by: wang.yuqi <yuqi.wang@daocloud.io>

tests/entrypoints/cohere/test_serving_conversion.py

@@ -511,7 +511,7 @@ class TestConvertToolMessage:
         # A list of text blocks maps 1-to-1 onto ``text`` content
         # parts, mirroring the shape ``_convert_user_message`` emits
         # for multi-part user content. vLLM's
-        # ``_parse_chat_message_content`` will collapse this back to
+        # ``parse_chat_message_content`` will collapse this back to
         # ``"\n".join(...)`` for the ``ConversationMessage`` handed to
         # renderers (see the tool-role branch in
         # ``vllm/entrypoints/chat_utils.py``), so downstream chat
@@ -533,7 +533,7 @@ class TestConvertToolMessage:
         """Document blocks must not survive on the OpenAI-shaped tool
         message.
 
-        vLLM's shared ``_parse_chat_message_content_parts`` runs a
+        vLLM's shared ``parse_chat_message_content_parts`` runs a
         Pydantic ``ValidatorIterator`` over the OpenAI content-part
         union, which only knows ``text`` / ``image_url`` / etc. and
         rejects ``{"type": "document", ...}`` blocks outright. We
@@ -588,7 +588,7 @@ class TestConvertToolMessage:
     def test_converted_tool_message_content_survives_openai_validation(self):
         """Regression for the reported crash.
 
-        ``vllm/entrypoints/chat_utils.py:_parse_chat_message_content_parts``
+        ``vllm/renders/chat_message_content.py:parse_chat_message_content_parts``
         runs a Pydantic ``ValidatorIterator`` over
         ``list[ChatCompletionContentPartParam]``, which is a
         discriminated union that only accepts ``text`` / ``image_url``

tests/entrypoints/unit_tests/test_chat_billing_header.py

@@ -6,7 +6,7 @@ from unittest.mock import MagicMock
 
 import pytest
 
-from vllm.renderers.chat_utils import _parse_chat_message_content
+from vllm.renderers.chat_utils import parse_chat_message_content
 
 HEADER = (
     "x-anthropic-billing-header: cc_version=2.1.220.300; "
@@ -17,7 +17,7 @@ HEADER = (
 def _parse(message, content_format="openai"):
     tracker = MagicMock()
     tracker.create_parser.return_value.mm_placeholder_storage.return_value = {}
-    return _parse_chat_message_content(
+    return parse_chat_message_content(
         message, tracker, content_format, interleave_strings=False
     )
 

tests/models/multimodal/processing/test_common.py

@@ -29,7 +29,7 @@ from vllm.multimodal.cache import MultiModalProcessorOnlyCache
 from vllm.multimodal.inputs import MultiModalFeatureSpec, batched_tensors_equal
 from vllm.multimodal.processing import BaseMultiModalProcessor, InputProcessingContext
 from vllm.platforms import current_platform
-from vllm.renderers.chat_utils import _get_embeds_data
+from vllm.renderers.chat_utils.item_tracker import _get_embeds_data
 from vllm.tokenizers import TokenizerLike, cached_tokenizer_from_config
 from vllm.utils.mistral import is_mistral_tokenizer
 

tests/renderers/test_chat_utils.py

@@ -16,24 +16,24 @@ from vllm.assets.video import VideoAsset
 from vllm.config import ModelConfig
 from vllm.exceptions import VLLMValidationError
 from vllm.inputs import MultiModalDataDict, MultiModalUUIDDict
+from vllm.multimodal.media import MEDIA_CONNECTOR_REGISTRY
 from vllm.multimodal.utils import (
     encode_audio_url,
     encode_image_url,
     encode_video_url,
 )
 from vllm.renderers.chat_utils import (
-    MEDIA_CONNECTOR_REGISTRY,
     AsyncMultiModalItemTracker,
     ChatCompletionMessageParam,
     ConversationMessage,
-    _load_embeds_dict,
-    _parse_chat_message_content,
-    _parse_metadata_array,
-    _postprocess_messages,
+    parse_chat_message_content,
     parse_chat_messages,
     parse_chat_messages_async,
     validate_chat_template,
 )
+from vllm.renderers.chat_utils.chat_message import _postprocess_messages
+from vllm.renderers.chat_utils.content_parser import _load_embeds_dict
+from vllm.renderers.chat_utils.item_tracker import _parse_metadata_array
 from vllm.utils.serial_utils import tensor2base64
 
 KIMI_K2_5_MODEL_ID = "moonshotai/Kimi-K2.5"
@@ -3205,7 +3205,7 @@ def _parse_one(message) -> ConversationMessage:
     parser = tracker.create_parser.return_value
     parser.model_config.enable_prompt_embeds = False
     parser.mm_placeholder_storage.return_value = {}
-    result = _parse_chat_message_content(
+    result = parse_chat_message_content(
         message, tracker, content_format="string", interleave_strings=False
     )
     assert len(result) == 1

tests/renderers/test_chat_utils_prompt_embeds.py

@@ -18,15 +18,16 @@ from transformers import AutoTokenizer
 
 from vllm.exceptions import VLLMValidationError
 from vllm.renderers import TokenizeParams
-from vllm.renderers.chat_utils import (
-    _ENABLE_PROMPT_EMBEDS_ERROR,
+from vllm.renderers.chat_utils import parse_chat_messages, parse_chat_messages_async
+from vllm.renderers.chat_utils.chat_message import (
     _PROMPT_EMBEDS_MISSING_DATA_ERROR,
     _RESERVED_PLACEHOLDER_IN_TEXT_ERROR,
     MM_PARSER_MAP,
+)
+from vllm.renderers.chat_utils.content_parser import (
+    _ENABLE_PROMPT_EMBEDS_ERROR,
     MODALITY_PLACEHOLDERS_MAP,
     PROMPT_EMBEDS_PLACEHOLDER_TOKEN,
-    parse_chat_messages,
-    parse_chat_messages_async,
 )
 from vllm.renderers.hf import (
     _PROMPT_EMBEDS_PLACEHOLDER_SPAN_MISMATCH_ERROR,

tests/renderers/test_warmup.py

@@ -28,9 +28,9 @@ from unittest.mock import MagicMock, patch
 
 import pytest
 
+from vllm.exceptions import ChatTemplateResolutionError
 from vllm.multimodal.processing import BaseMultiModalProcessor
 from vllm.renderers.base import BaseRenderer
-from vllm.renderers.chat_utils import ChatTemplateResolutionError
 from vllm.renderers.params import ChatParams
 
 

vllm/entrypoints/cohere/serving.py

@@ -502,7 +502,7 @@ class CohereServingChatV2(OpenAIServingChat):
         # Map every content block to a plain ``{"type": "text", "text":
         # ...}`` part, mirroring the shape :meth:`_convert_user_message`
         # emits for multi-part user content. vLLM's
-        # ``_parse_chat_message_content`` collapses a text-only tool
+        # ``parse_chat_message_content`` collapses a text-only tool
         # part list back into ``"\n".join(...)`` on the way to the
         # renderer, so simple chat templates that expect string tool
         # content still work; keeping the list here just preserves the
@@ -512,7 +512,7 @@ class CohereServingChatV2(OpenAIServingChat):
         # Tool message content uses ``ToolMessageV2Content``, a union of
         # ``TextToolContent`` and ``DocumentToolContent``. The latter
         # produces ``{"type": "document", "document": {...}}`` blocks,
-        # which vLLM's shared ``_parse_chat_message_content_parts`` in
+        # which vLLM's shared ``parse_chat_message_content_parts`` in
         # ``vllm/renderers/chat_utils.py`` rejects because the OpenAI
         # ``ChatCompletionContentPartParam`` union only knows text /
         # image / audio / etc. Emitting them anyway triggers a Pydantic

vllm/entrypoints/pooling/embed/io_processor.py

@@ -9,7 +9,6 @@ from openai.types.chat import (
     ChatCompletionContentPartImageParam,
     ChatCompletionContentPartTextParam,
 )
-from openai.types.chat.chat_completion_content_part_image_param import ImageURL
 
 from vllm import PoolingParams
 from vllm.exceptions import VLLMValidationError
@@ -21,6 +20,7 @@ from vllm.renderers.chat_utils import (
     ChatCompletionContentPartParam,
     ChatCompletionMessageParam,
     CustomChatCompletionMessageParam,
+    ImageURL,
 )
 from vllm.renderers.hf import resolve_chat_template
 from vllm.utils.collection_utils import chunk_list

vllm/entrypoints/pooling/scoring/io_processor.py

@@ -7,9 +7,9 @@ from typing import Any, TypeAlias, cast
 import torch.nn.functional as F
 
 from vllm import PoolingParams, PoolingRequestOutput, TokensPrompt
+from vllm.exceptions import ChatTemplateResolutionError
 from vllm.logger import init_logger
 from vllm.renderers import TokenizeParams
-from vllm.renderers.chat_utils import ChatTemplateResolutionError
 from vllm.renderers.hf import safe_apply_chat_template
 from vllm.renderers.inputs.preprocess import (
     extract_target_prompt,

vllm/entrypoints/pooling/scoring/utils.py

@@ -15,7 +15,7 @@ from vllm.renderers.chat_utils import (
     ChatCompletionContentPartTextParam,
     ConversationMessage,
     MultiModalItemTracker,
-    _parse_chat_message_content_parts,
+    parse_chat_message_content_parts,
 )
 
 from .typing import (
@@ -193,7 +193,7 @@ def _parse_score_content(
 
     mm_parser = mm_tracker.create_parser()
 
-    parse_res = _parse_chat_message_content_parts(
+    parse_res = parse_chat_message_content_parts(
         role=role,
         parts=parts,
         mm_tracker=mm_tracker,

vllm/exceptions.py

@@ -62,6 +62,12 @@ class VLLMNotFoundError(VLLMClientError):
     pass
 
 
+class ChatTemplateResolutionError(VLLMClientError):
+    """Raised when chat template resolution fails."""
+
+    pass
+
+
 class LoRAAdapterNotFoundError(VLLMNotFoundError):
     """Exception raised when a LoRA adapter is not found.
 

vllm/renderers/base.py

@@ -362,7 +362,7 @@ class BaseRenderer(ABC, Generic[_T]):
         no-op if the background warmup — or a prior reset_mm_cache that joined
         it — already ran it.
         """
-        from vllm.renderers.chat_utils import ChatTemplateResolutionError
+        from vllm.exceptions import ChatTemplateResolutionError
 
         # prevent MM processor hangs
         with set_default_torch_num_threads(1):

vllm/renderers/chat_utils/__init__.py

@@ -0,0 +1,42 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+from .chat_message import (
+    has_non_text_content,
+    parse_chat_message_content,
+    parse_chat_message_content_parts,
+    parse_chat_messages,
+    parse_chat_messages_async,
+)
+from .chat_template import (
+    ChatTemplateConfig,
+    load_chat_template,
+    validate_chat_template,
+)
+from .content_parser import PROMPT_EMBEDS_PLACEHOLDER_TOKEN
+from .item_tracker import (
+    AsyncMultiModalItemTracker,
+    BaseMultiModalItemTracker,
+    MultiModalItemTracker,
+)
+from .tool_call import get_tool_call_id_type, make_tool_call_id
+from .types import *
+from .types import __all__ as all_types
+
+__all__ = [
+    *all_types,
+    "ChatTemplateConfig",
+    "load_chat_template",
+    "validate_chat_template",
+    "get_tool_call_id_type",
+    "make_tool_call_id",
+    "BaseMultiModalItemTracker",
+    "MultiModalItemTracker",
+    "AsyncMultiModalItemTracker",
+    "has_non_text_content",
+    "parse_chat_message_content",
+    "parse_chat_message_content_parts",
+    "parse_chat_messages",
+    "parse_chat_messages_async",
+    "PROMPT_EMBEDS_PLACEHOLDER_TOKEN",
+]

vllm/renderers/chat_utils/chat_message.py

@@ -0,0 +1,826 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+import json
+import types
+from collections import Counter
+from collections.abc import Callable, Iterable
+from functools import partial
+from typing import Any, Final, TypeAlias, Union, cast, get_args, get_origin
+
+from openai.types.chat import (
+    ChatCompletionAssistantMessageParam,
+    ChatCompletionContentPartImageParam,
+    ChatCompletionContentPartInputAudioParam,
+    ChatCompletionContentPartRefusalParam,
+    ChatCompletionToolMessageParam,
+)
+from openai.types.responses import ResponseInputImageParam
+from PIL import Image
+from pydantic import TypeAdapter
+
+from vllm.config import ModelConfig
+from vllm.exceptions import VLLMValidationError
+from vllm.inputs import MultiModalDataDict, MultiModalUUIDDict
+from vllm.logger import init_logger
+
+from .content_parser import (
+    MODALITY_PLACEHOLDERS_MAP,
+    PROMPT_EMBEDS_PLACEHOLDER_TOKEN,
+    BaseMultiModalContentParser,
+)
+from .item_tracker import (
+    AsyncMultiModalItemTracker,
+    BaseMultiModalItemTracker,
+    MultiModalItemTracker,
+)
+from .types import (
+    ChatCompletionContentPartAudioEmbedsParam,
+    ChatCompletionContentPartAudioParam,
+    ChatCompletionContentPartImageEmbedsParam,
+    ChatCompletionContentPartParam,
+    ChatCompletionContentPartPromptEmbedsParam,
+    ChatCompletionContentPartTextParam,
+    ChatCompletionContentPartVideoEmbedsParam,
+    ChatCompletionContentPartVideoParam,
+    ChatCompletionMessageParam,
+    ChatTemplateContentFormat,
+    ConversationMessage,
+    CustomChatCompletionContentPILImageParam,
+    CustomChatCompletionContentSimpleAudioParam,
+    CustomChatCompletionContentSimpleImageParam,
+    CustomChatCompletionContentSimpleVideoParam,
+    CustomChatCompletionContentToolReferenceParam,
+    CustomThinkCompletionContentParam,
+    InputAudio,
+    MultiModalEmbedsPayload,
+    PILImage,
+)
+
+logger = init_logger(__name__)
+
+
+_PROMPT_EMBEDS_MISSING_DATA_ERROR: Final[str] = (
+    "prompt_embeds content part requires a non-empty `data` field "
+    "with base64-encoded tensor bytes."
+)
+
+_RESERVED_PLACEHOLDER_IN_TEXT_ERROR: Final[str] = (
+    "Text content may not contain the reserved placeholder {token!r}. "
+    "This placeholder is used internally to mark `prompt_embeds` splice "
+    "positions in the tokenized prompt."
+)
+
+
+# No need to validate using Pydantic again
+_TextParser = partial(cast, ChatCompletionContentPartTextParam)
+_ImageEmbedsParser = partial(cast, ChatCompletionContentPartImageEmbedsParam)
+_AudioEmbedsParser = partial(cast, ChatCompletionContentPartAudioEmbedsParam)
+_VideoEmbedsParser = partial(cast, ChatCompletionContentPartVideoEmbedsParam)
+_PromptEmbedsParser = partial(cast, ChatCompletionContentPartPromptEmbedsParam)
+_InputAudioParser = partial(cast, ChatCompletionContentPartInputAudioParam)
+_RefusalParser = partial(cast, ChatCompletionContentPartRefusalParam)
+_PILImageParser = partial(cast, CustomChatCompletionContentPILImageParam)
+_ThinkParser = partial(cast, CustomThinkCompletionContentParam)
+_AssistantParser = partial(cast, ChatCompletionAssistantMessageParam)
+_ToolParser = partial(cast, ChatCompletionToolMessageParam)
+
+# Need to validate url objects
+_ImageParser = TypeAdapter(ChatCompletionContentPartImageParam).validate_python
+_AudioParser = TypeAdapter(ChatCompletionContentPartAudioParam).validate_python
+_VideoParser = TypeAdapter(ChatCompletionContentPartVideoParam).validate_python
+
+_ResponsesInputImageParser = TypeAdapter(ResponseInputImageParam).validate_python
+ContentPart: TypeAlias = (
+    MultiModalEmbedsPayload | dict[str, str] | InputAudio | PILImage
+)
+
+# Define a mapping from part types to their corresponding parsing functions.
+MM_PARSER_MAP: dict[
+    str,
+    Callable[[ChatCompletionContentPartParam], ContentPart],
+] = {
+    "text": lambda part: _TextParser(part).get("text", None),
+    "thinking": lambda part: _ThinkParser(part).get("thinking", None),
+    "input_text": lambda part: _TextParser(part).get("text", None),
+    "output_text": lambda part: _TextParser(part).get("text", None),
+    "input_image": lambda part: _ResponsesInputImageParser(part).get("image_url", None),
+    "image_url": lambda part: _ImageParser(part).get("image_url", {}).get("url", None),
+    "image_embeds": lambda part: _ImageEmbedsParser(part).get("image_embeds", None),
+    "audio_embeds": lambda part: _AudioEmbedsParser(part).get("audio_embeds", None),
+    "video_embeds": lambda part: _VideoEmbedsParser(part).get("video_embeds", None),
+    "prompt_embeds": lambda part: _PromptEmbedsParser(part).get("data", None),
+    "image_pil": lambda part: _PILImageParser(part).get("image_pil", None),
+    "audio_url": lambda part: _AudioParser(part).get("audio_url", {}).get("url", None),
+    "input_audio": lambda part: _InputAudioParser(part).get("input_audio", None),
+    "refusal": lambda part: _RefusalParser(part).get("refusal", None),
+    "video_url": lambda part: _VideoParser(part).get("video_url", {}).get("url", None),
+    "tool_reference": lambda part: cast(
+        CustomChatCompletionContentToolReferenceParam, part
+    ).get("name", None),
+}
+
+
+def _get_interleaved_text_prompt(
+    placeholder_storage: dict[str, list], texts: list[str]
+) -> str:
+    for idx, elem in enumerate(texts):
+        if elem in placeholder_storage:
+            slots = placeholder_storage[elem]
+            if not slots:
+                raise VLLMValidationError(
+                    f"Found more '{elem}' placeholders in input prompt than "
+                    "actual multimodal data items."
+                )
+            texts[idx] = slots.pop(0)
+
+    return "\n".join(texts)
+
+
+# TODO: Let user specify how to insert multimodal tokens into prompt
+# (similar to chat template)
+def get_full_multimodal_text_prompt(
+    placeholder_storage: dict[str, list],
+    texts: list[str],
+    interleave_strings: bool,
+    multimodal_content_part_separator: str = "\n",
+) -> str:
+    """Combine multimodal prompts for a multimodal language model."""
+    # flatten storage to make it looks like
+    # {
+    #   "<|image|>": 2,
+    #   "<|audio|>": 1
+    # }
+    placeholder_counts = Counter(
+        [v for elem in placeholder_storage.values() for v in elem]
+    )
+
+    if interleave_strings:
+        text_prompt = _get_interleaved_text_prompt(placeholder_storage, texts)
+    else:
+        text_prompt = "\n".join(texts)
+
+    # Pass interleaved text further in case the user used image placeholders
+    # himself, but forgot to disable the 'interleave_strings' flag
+
+    # Look through the text prompt to check for missing placeholders
+    missing_placeholders: list[str] = []
+    for placeholder in placeholder_counts:
+        # For any existing placeholder in the text prompt, we leave it as is
+        placeholder_counts[placeholder] -= text_prompt.count(placeholder)
+
+        if placeholder_counts[placeholder] < 0:
+            logger.error(
+                "Placeholder count is negative! "
+                "Ensure that the 'interleave_strings' flag is disabled "
+                "(current value: %s) "
+                "when manually placing image placeholders.",
+                interleave_strings,
+            )
+            logger.debug("Input prompt: %s", text_prompt)
+            raise VLLMValidationError(
+                f"Found more '{placeholder}' placeholders in input prompt than "
+                "actual multimodal data items.",
+                parameter="messages",
+            )
+
+        missing_placeholders.extend([placeholder] * placeholder_counts[placeholder])
+
+    # NOTE: Default behaviour: we always add missing placeholders
+    # at the front of the prompt, if interleave_strings=False
+    if text_prompt:
+        return multimodal_content_part_separator.join(
+            missing_placeholders + [text_prompt]
+        )
+    else:
+        return multimodal_content_part_separator.join(missing_placeholders)
+
+
+def _collect_known_content_part_fields() -> frozenset[str]:
+    fields: set[str] = set()
+    stack: list[Any] = [ChatCompletionContentPartParam]
+    while stack:
+        node = stack.pop()
+        if get_origin(node) in (Union, types.UnionType):
+            stack.extend(get_args(node))
+        elif hasattr(node, "__required_keys__"):
+            fields |= node.__required_keys__ | node.__optional_keys__
+    return frozenset(fields)
+
+
+_KNOWN_CONTENT_PART_FIELDS = _collect_known_content_part_fields()
+
+
+def _collect_extra_fields(part: dict[str, Any]) -> dict[str, Any]:
+    return {k: v for k, v in part.items() if k not in _KNOWN_CONTENT_PART_FIELDS}
+
+
+def parse_chat_message_content_mm_part(
+    part: ChatCompletionContentPartParam,
+) -> tuple[str, ContentPart]:
+    """Parses a given multi-modal content part based on its type.
+
+    Args:
+        part: A dict containing the content part, with a potential 'type' field.
+
+    Returns:
+        A tuple (part_type, content) where:
+        - part_type: Type of the part (e.g., 'text', 'image_url').
+        - content: Parsed content (e.g., text, image URL).
+
+    Raises:
+        ValueError: If the 'type' field is missing and no direct URL is found.
+
+    """
+    assert isinstance(
+        part, dict
+    )  # This is needed to avoid mypy errors: part.get() from str
+    part_type = part.get("type", None)
+    uuid = part.get("uuid", None)
+
+    if isinstance(part_type, str) and part_type in MM_PARSER_MAP and uuid is None:  # noqa: E501
+        content = MM_PARSER_MAP[part_type](part)
+
+        # Special case for 'image_url.detail'
+        # We only support 'auto', which is the default
+        if part_type == "image_url" and part.get("detail", "auto") != "auto":
+            logger.warning(
+                "'image_url.detail' is currently not supported and will be ignored."
+            )
+
+        return part_type, content
+
+    # Handle missing 'type' but provided direct URL fields.
+    # 'type' is required field by pydantic
+    if part_type is None or uuid is not None:
+        if "image_url" in part:
+            image_params = cast(CustomChatCompletionContentSimpleImageParam, part)
+            image_url = image_params.get("image_url", None)
+            if isinstance(image_url, dict):
+                # Can potentially happen if user provides a uuid
+                # with url as a dict of {"url": url}
+                image_url = image_url.get("url", None)
+            return "image_url", image_url
+        if "image_pil" in part:
+            # "image_pil" could be None if UUID is provided.
+            image_params = cast(  # type: ignore
+                CustomChatCompletionContentPILImageParam, part
+            )
+            image_pil = image_params.get("image_pil", None)
+            return "image_pil", image_pil
+        if "image_embeds" in part:
+            # "image_embeds" could be None if UUID is provided.
+            image_params = cast(  # type: ignore
+                ChatCompletionContentPartImageEmbedsParam, part
+            )
+            image_embeds = image_params.get("image_embeds", None)
+            return "image_embeds", image_embeds
+        if "audio_embeds" in part:
+            # "audio_embeds" could be None if UUID is provided.
+            audio_params = cast(  # type: ignore[assignment]
+                ChatCompletionContentPartAudioEmbedsParam, part
+            )
+            audio_embeds = audio_params.get("audio_embeds", None)
+            return "audio_embeds", audio_embeds
+        if "video_embeds" in part:
+            # "video_embeds" could be None if UUID is provided.
+            video_embeds_params = cast(  # type: ignore[assignment]
+                ChatCompletionContentPartVideoEmbedsParam, part
+            )
+            video_embeds = video_embeds_params.get("video_embeds", None)
+            return "video_embeds", video_embeds
+        if "prompt_embeds" in part:
+            prompt_embeds_params = cast(  # type: ignore[assignment]
+                ChatCompletionContentPartPromptEmbedsParam, part
+            )
+            return "prompt_embeds", prompt_embeds_params.get("data", None)
+        if "audio_url" in part:
+            audio_params = cast(  # type: ignore[assignment]
+                CustomChatCompletionContentSimpleAudioParam, part
+            )
+            audio_url = audio_params.get("audio_url", None)
+            if isinstance(audio_url, dict):
+                # Can potentially happen if user provides a uuid
+                # with url as a dict of {"url": url}
+                audio_url = audio_url.get("url", None)
+            return "audio_url", audio_url
+        if "input_audio" in part:
+            input_audio_params = _InputAudioParser(part).get("input_audio", None)
+            return "input_audio", input_audio_params
+        if "video_url" in part:
+            video_params = cast(CustomChatCompletionContentSimpleVideoParam, part)
+            video_url = video_params.get("video_url", None)
+            if isinstance(video_url, dict):
+                # Can potentially happen if user provides a uuid
+                # with url as a dict of {"url": url}
+                video_url = video_url.get("url", None)
+            return "video_url", video_url
+        if "tool_reference" in part:
+            tool_reference_params = cast(
+                CustomChatCompletionContentToolReferenceParam, part
+            )
+            tool_reference = tool_reference_params.get("name", None)
+            return "tool_reference", tool_reference
+        # Raise an error if no 'type' or direct URL is found.
+        raise VLLMValidationError(
+            "Missing 'type' field in multimodal part.", parameter="type"
+        )
+
+    if not isinstance(part_type, str):
+        raise VLLMValidationError(
+            "Invalid 'type' field in multimodal part.", parameter="type"
+        )
+    return part_type, "unknown part_type content"
+
+
+PART_TYPES_TO_SKIP_NONE_CONTENT = (
+    "text",
+    "refusal",
+)
+
+# Content part types parsed as text rather than multimodal data.
+TEXT_PART_TYPES = frozenset(
+    {"text", "input_text", "output_text", "refusal", "thinking"}
+)
+# Content part types that carry no multimodal data.
+_TEXT_CONTENT_PART_TYPES = TEXT_PART_TYPES | {"tool_reference"}
+# Keys that mark a content part as multimodal, whatever its ``type``.
+_MEDIA_CONTENT_PART_KEYS = frozenset(MM_PARSER_MAP) - _TEXT_CONTENT_PART_TYPES
+
+
+def has_non_text_content(messages: Any) -> bool:
+    """Whether any message in ``messages`` has a non-text content part.
+
+    Only list content is inspected, so validated chat content that is a
+    one-shot iterator is never consumed.
+    """
+    if not isinstance(messages, list):
+        return False
+    for msg in messages:
+        content = msg.get("content") if isinstance(msg, dict) else None
+        if not isinstance(content, list):
+            continue
+        for part in content:
+            if isinstance(part, dict) and (
+                any(key in part for key in _MEDIA_CONTENT_PART_KEYS)
+                or not isinstance(part_type := part.get("type", "text"), str)
+                or part_type not in _TEXT_CONTENT_PART_TYPES
+            ):
+                return True
+    return False
+
+
+def parse_chat_message_content_parts(
+    role: str,
+    parts: Iterable[ChatCompletionContentPartParam],
+    mm_tracker: BaseMultiModalItemTracker,
+    *,
+    wrap_dicts: bool,
+    interleave_strings: bool,
+    mm_processor_kwargs: dict[str, Any] | None = None,
+    multimodal_content_part_separator="\n",
+) -> list[ConversationMessage]:
+    content = list[ContentPart]()
+
+    mm_parser = mm_tracker.create_parser(mm_processor_kwargs=mm_processor_kwargs)
+
+    for part in parts:
+        parse_res = parse_chat_message_content_part(
+            part,
+            mm_parser,
+            wrap_dicts=wrap_dicts,
+            interleave_strings=interleave_strings,
+        )
+        if parse_res:
+            content.append(parse_res)
+
+    if wrap_dicts:
+        # Parsing wraps images and texts as interleaved dictionaries
+        return [ConversationMessage(role=role, content=content)]  # type: ignore
+    texts = cast(list[str], content)
+    mm_placeholder_storage = mm_parser.mm_placeholder_storage()
+    if mm_placeholder_storage:
+        text_prompt = get_full_multimodal_text_prompt(
+            mm_placeholder_storage,
+            texts,
+            interleave_strings,
+            multimodal_content_part_separator=multimodal_content_part_separator,
+        )
+    else:
+        text_prompt = "\n".join(texts)
+
+    return [ConversationMessage(role=role, content=text_prompt)]
+
+
+def _reject_reserved_placeholder_in_text(text: str, model_config: ModelConfig) -> None:
+    """Reject user-supplied text parts that contains the reserved `prompt_embeds`
+    placeholder sentinel.
+
+    When the server accepts `prompt_embeds`, the placeholder token is
+    registered as a single unsplittable special token on the tokenizer. Any
+    user text that happens to contain the literal sequence would tokenize to
+    the same ID and be mistaken for a splice point by the renderer, letting a
+    caller move or inject splice positions via plain text content.
+    """
+    if model_config.enable_prompt_embeds and PROMPT_EMBEDS_PLACEHOLDER_TOKEN in text:
+        raise VLLMValidationError(
+            _RESERVED_PLACEHOLDER_IN_TEXT_ERROR.format(
+                token=PROMPT_EMBEDS_PLACEHOLDER_TOKEN
+            ),
+            parameter="messages",
+        )
+
+
+def parse_chat_message_content_part(
+    part: ChatCompletionContentPartParam,
+    mm_parser: BaseMultiModalContentParser,
+    *,
+    wrap_dicts: bool,
+    interleave_strings: bool,
+) -> ContentPart | None:
+    """Parses a single part of a conversation. If wrap_dicts is True,
+    structured dictionary pieces for texts and images will be
+    wrapped in dictionaries, i.e., {"type": "text", "text", ...} and
+    {"type": "image"}, respectively. Otherwise multimodal data will be
+    handled by mm_parser, and texts will be returned as strings to be joined
+    with multimodal placeholders.
+    """
+    if isinstance(part, str):  # Handle plain text parts
+        _reject_reserved_placeholder_in_text(part, mm_parser.model_config)
+        if wrap_dicts:
+            return {"type": "text", "text": part}
+        return part
+    # Handle structured dictionary parts
+    part_type, content = parse_chat_message_content_mm_part(part)
+    # if part_type is text/refusal/image_url/audio_url/video_url/input_audio but
+    # content is None, log a warning and skip
+    if part_type in PART_TYPES_TO_SKIP_NONE_CONTENT and content is None:
+        logger.warning(
+            "Skipping multimodal part '%s' (type: '%s') "
+            "with empty / unparsable content.",
+            part,
+            part_type,
+        )
+        return None
+
+    if part_type in TEXT_PART_TYPES:
+        str_content = cast(str, content)
+        _reject_reserved_placeholder_in_text(str_content, mm_parser.model_config)
+        if wrap_dicts:
+            result: dict[str, Any] = {"type": "text", "text": str_content}
+            result.update(_collect_extra_fields(cast(dict[str, Any], part)))
+            return result
+        else:
+            return str_content
+
+    # For media items, if a user has provided one, use it. Otherwise, insert
+    # a placeholder empty uuid.
+    uuid = part.get("uuid", None)
+    if uuid is not None:
+        uuid = str(uuid)
+
+    modality = None
+    if part_type == "image_pil":
+        image_content = cast(Image.Image, content) if content is not None else None
+        mm_parser.parse_image_pil(image_content, uuid)
+        modality = "image"
+    elif part_type in ("image_url", "input_image"):
+        str_content = cast(str, content)
+        mm_parser.parse_image(str_content, uuid)
+        modality = "image"
+    elif part_type == "image_embeds":
+        content = (
+            cast(MultiModalEmbedsPayload, content) if content is not None else None
+        )
+        mm_parser.parse_image_embeds(content, uuid)
+        modality = "image"
+    elif part_type == "audio_embeds":
+        content = (
+            cast(MultiModalEmbedsPayload, content) if content is not None else None
+        )
+        mm_parser.parse_audio_embeds(content, uuid)
+        modality = "audio"
+    elif part_type == "video_embeds":
+        content = (
+            cast(MultiModalEmbedsPayload, content) if content is not None else None
+        )
+        mm_parser.parse_video_embeds(content, uuid)
+        modality = "video"
+    elif part_type == "prompt_embeds":
+        if not content:
+            raise VLLMValidationError(
+                _PROMPT_EMBEDS_MISSING_DATA_ERROR, parameter="prompt_embeds"
+            )
+        mm_parser.parse_prompt_embeds(cast(str, content))
+        modality = "prompt_embeds"
+    elif part_type == "audio_url":
+        str_content = cast(str, content)
+        mm_parser.parse_audio(str_content, uuid)
+        modality = "audio"
+    elif part_type == "input_audio":
+        dict_content = cast(InputAudio, content)
+        mm_parser.parse_input_audio(dict_content, uuid)
+        modality = "audio"
+    elif part_type == "video_url":
+        str_content = cast(str, content)
+        mm_parser.parse_video(str_content, uuid)
+        modality = "video"
+    elif part_type == "tool_reference":
+        # Tool references are not multimodal data — they reference deferred
+        # tools and are passed through as-is for the chat template to expand.
+        if wrap_dicts:
+            return {"type": "tool_reference", "name": cast(str, content)}
+        return cast(str, content)
+    else:
+        supported = sorted(MM_PARSER_MAP.keys() | set(PART_TYPES_TO_SKIP_NONE_CONTENT))
+        raise VLLMValidationError(
+            f"Unsupported chat content part type: {part_type!r}. "
+            f"Supported types: {', '.join(supported)}.",
+            parameter="type",
+            value=part_type,
+        )
+
+    if wrap_dicts:
+        if modality == "prompt_embeds":
+            # Chat templates don't know about the "prompt_embeds" modality,
+            # emit the single sentinel token as text so the template renders
+            # it inline. The renderer later expands it to N tokens post-tokenize.
+            return {"type": "text", "text": PROMPT_EMBEDS_PLACEHOLDER_TOKEN}
+        result = {"type": modality}
+        result.update(_collect_extra_fields(cast(dict[str, Any], part)))
+        return result
+    if modality == "prompt_embeds":
+        # Emit the renderer token inline regardless of `interleave_strings`,
+        # prompt_embeds are spliced at the token offset so position matters.
+        # Falling back to front-padding via `missing_placeholders` would
+        # reorder them relative to surrounding text.
+        return PROMPT_EMBEDS_PLACEHOLDER_TOKEN
+    return MODALITY_PLACEHOLDERS_MAP[modality] if interleave_strings else None
+
+
+_CLAUDE_CODE_BILLING_HEADER = "x-anthropic-billing-header"
+
+
+def _strip_claude_code_billing_header(
+    parts: Iterable[ChatCompletionContentPartParam],
+) -> list[ChatCompletionContentPartParam]:
+    """Drop Claude Code's attribution header line from system text parts.
+
+    Its ``cch`` value changes on every request, so a prompt that keeps it
+    misses the prefix cache from the header on. Gateways that translate
+    Anthropic requests to chat completions keep it; ``/v1/messages`` drops
+    the same block.
+    """
+    out: list[ChatCompletionContentPartParam] = []
+    for part in parts:
+        if isinstance(part, str):
+            text: object = part
+        elif isinstance(part, dict) and part.get("type") == "text":
+            text = part.get("text")
+        else:
+            text = None
+        if not (isinstance(text, str) and text.startswith(_CLAUDE_CODE_BILLING_HEADER)):
+            out.append(part)
+            continue
+        rest = text.partition("\n")[2]
+        if not rest:
+            continue
+        out.append(
+            cast(ChatCompletionContentPartParam, rest)
+            if isinstance(part, str)
+            else cast(ChatCompletionContentPartParam, {**part, "text": rest})
+        )
+    return out
+
+
+def parse_chat_message_content(
+    message: ChatCompletionMessageParam,
+    mm_tracker: BaseMultiModalItemTracker,
+    content_format: ChatTemplateContentFormat,
+    interleave_strings: bool,
+    mm_processor_kwargs: dict[str, Any] | None = None,
+) -> list[ConversationMessage]:
+    role = message["role"]
+    content = message.get("content")
+    reasoning = message.get("reasoning")
+
+    if content is None:
+        content = []
+    elif isinstance(content, str):
+        content = [ChatCompletionContentPartTextParam(type="text", text=content)]
+    if role == "system":
+        content = _strip_claude_code_billing_header(content)  # type: ignore[arg-type]
+    result = parse_chat_message_content_parts(
+        role,
+        content,  # type: ignore
+        mm_tracker,
+        wrap_dicts=(content_format == "openai"),
+        interleave_strings=interleave_strings,
+        mm_processor_kwargs=mm_processor_kwargs,
+    )
+
+    for result_msg in result:
+        if role == "assistant":
+            parsed_msg = _AssistantParser(message)
+
+            # The 'tool_calls' is not None check ensures compatibility.
+            # It's needed only if downstream code doesn't strictly
+            # follow the OpenAI spec.
+            if "tool_calls" in parsed_msg and parsed_msg["tool_calls"] is not None:
+                result_msg["tool_calls"] = list(parsed_msg["tool_calls"])
+            # Include reasoning if present for interleaved thinking.
+            if reasoning is not None:
+                result_msg["reasoning"] = cast(str, reasoning)
+                result_msg["reasoning_content"] = cast(
+                    str, reasoning
+                )  # keep compatibility
+        elif role == "tool":
+            parsed_msg = _ToolParser(message)
+            if "tool_call_id" in parsed_msg:
+                result_msg["tool_call_id"] = parsed_msg["tool_call_id"]
+            # Normalize tool message content from OpenAI array format to plain
+            # string. Clients like Claude Code / Cursor send tool results as
+            # [{"type": "text", "text": "..."}], but most chat templates only
+            # handle string content for tool messages.
+            # However, tool_reference items must be preserved as structured
+            # dicts for the chat template to expand them.
+            msg_content = result_msg.get("content")
+            if isinstance(msg_content, list):
+                has_non_text = any(
+                    isinstance(item, dict) and item.get("type") != "text"
+                    for item in msg_content
+                )
+                if has_non_text:
+                    # Keep structured content (e.g., tool_reference)
+                    result_msg["content"] = msg_content
+                else:
+                    texts = [
+                        item.get("text", "")
+                        for item in msg_content
+                        if isinstance(item, dict) and item.get("type") == "text"
+                    ]
+                    result_msg["content"] = "\n".join(texts) if texts else ""
+
+        if "name" in message and isinstance(message["name"], str):
+            result_msg["name"] = message["name"]
+
+        if "task" in message and isinstance(message["task"], str):
+            result_msg["task"] = message["task"]
+
+        if role == "developer":
+            result_msg["tools"] = message.get("tools", None)
+    return result
+
+
+def _postprocess_messages(messages: list[ConversationMessage]) -> None:
+    # per the Transformers docs & maintainers, tool call arguments in
+    # assistant-role messages with tool_calls need to be dicts not JSON str -
+    # this is how tool-use chat templates will expect them moving forwards
+    # so, for messages that have tool_calls, parse the string (which we get
+    # from openAI format) to dict
+    for message in messages:
+        if message["role"] == "assistant" and "tool_calls" in message:
+            tool_calls = message.get("tool_calls")
+            if not isinstance(tool_calls, list):
+                continue
+
+            if len(tool_calls) == 0:
+                # Drop empty tool_calls to keep templates on the normal assistant path.
+                message.pop("tool_calls", None)
+                continue
+
+            for item in tool_calls:
+                if not isinstance(item, dict):
+                    raise VLLMValidationError(
+                        "assistant tool_calls entries must be objects.",
+                        parameter="tool_calls",
+                    )
+
+                function = item.get("function")
+                if item.get("type", "function") != "function" or not isinstance(
+                    function, dict
+                ):
+                    raise VLLMValidationError(
+                        "chat completions only support assistant tool_calls "
+                        "of type 'function'.",
+                        parameter="tool_calls",
+                    )
+
+                # if arguments is None or empty string, set to {}
+                if content := function.get("arguments"):
+                    if isinstance(content, dict):
+                        parsed = content
+                    else:
+                        if isinstance(content, str):
+                            try:
+                                parsed = json.loads(content)
+                            except json.JSONDecodeError:
+                                # A malformed `arguments` string lives in
+                                # conversation history, so failing the request
+                                # here would fail every subsequent turn too and
+                                # leave the conversation unrecoverable. Coerce
+                                # to an empty object so the turn can proceed.
+                                logger.warning(
+                                    "Tool call %r has arguments that are not valid "
+                                    "JSON (%d chars); coercing to an empty object "
+                                    "so the conversation can continue.",
+                                    function.get("name"),
+                                    len(content),
+                                )
+                                parsed = None
+                        else:
+                            parsed = content
+
+                        if not isinstance(parsed, dict):
+                            if parsed is not None:
+                                # Valid JSON, but not an object (e.g. "[]",
+                                # "42", "true").
+                                # Chat templates require a mapping.
+                                logger.warning(
+                                    "Tool call %r arguments decoded to %s, not a "
+                                    "JSON object; coercing to an empty object.",
+                                    function.get("name"),
+                                    type(parsed).__name__,
+                                )
+                            parsed = {}
+
+                    function["arguments"] = parsed
+                else:
+                    function["arguments"] = {}
+
+
+def parse_chat_messages(
+    messages: list[ChatCompletionMessageParam],
+    model_config: ModelConfig,
+    content_format: ChatTemplateContentFormat,
+    media_io_kwargs: dict[str, dict[str, Any]] | None = None,
+    mm_processor_kwargs: dict[str, Any] | None = None,
+) -> tuple[
+    list[ConversationMessage],
+    MultiModalDataDict | None,
+    MultiModalUUIDDict | None,
+]:
+    conversation: list[ConversationMessage] = []
+    mm_tracker = MultiModalItemTracker(
+        model_config,
+        media_io_kwargs=media_io_kwargs,
+    )
+
+    for msg in messages:
+        sub_messages = parse_chat_message_content(
+            msg,
+            mm_tracker,
+            content_format,
+            interleave_strings=(
+                content_format == "string"
+                and model_config.multimodal_config is not None
+                and model_config.multimodal_config.interleave_mm_strings
+            ),
+            mm_processor_kwargs=mm_processor_kwargs,
+        )
+
+        conversation.extend(sub_messages)
+
+    _postprocess_messages(conversation)
+
+    mm_data, mm_uuids = mm_tracker.resolve_items()
+
+    return conversation, mm_data, mm_uuids
+
+
+async def parse_chat_messages_async(
+    messages: list[ChatCompletionMessageParam],
+    model_config: ModelConfig,
+    content_format: ChatTemplateContentFormat,
+    media_io_kwargs: dict[str, dict[str, Any]] | None = None,
+    mm_processor_kwargs: dict[str, Any] | None = None,
+) -> tuple[
+    list[ConversationMessage],
+    MultiModalDataDict | None,
+    MultiModalUUIDDict | None,
+]:
+    conversation: list[ConversationMessage] = []
+    mm_tracker = AsyncMultiModalItemTracker(
+        model_config,
+        media_io_kwargs=media_io_kwargs,
+    )
+
+    for msg in messages:
+        sub_messages = parse_chat_message_content(
+            msg,
+            mm_tracker,
+            content_format,
+            interleave_strings=(
+                content_format == "string"
+                and model_config.multimodal_config is not None
+                and model_config.multimodal_config.interleave_mm_strings
+            ),
+            mm_processor_kwargs=mm_processor_kwargs,
+        )
+
+        conversation.extend(sub_messages)
+
+    _postprocess_messages(conversation)
+
+    mm_data, mm_uuids = await mm_tracker.resolve_items()
+
+    return conversation, mm_data, mm_uuids

vllm/renderers/chat_utils/chat_template.py

@@ -0,0 +1,112 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+from dataclasses import dataclass
+from functools import lru_cache
+from pathlib import Path
+
+from vllm.exceptions import VLLMValidationError
+
+from .types import ChatTemplateContentFormatOption
+
+
+@dataclass
+class ChatTemplateConfig:
+    chat_template: str | None = None
+    chat_template_content_format: ChatTemplateContentFormatOption = "auto"
+    trust_request_chat_template: bool = False
+    trust_request_mm_kwargs: bool = False
+
+
+def validate_chat_template(chat_template: Path | str | None):
+    """Raises if the provided chat template appears invalid."""
+    if chat_template is None:
+        return
+
+    elif isinstance(chat_template, Path) and not chat_template.exists():
+        raise FileNotFoundError("the supplied chat template path doesn't exist")
+
+    elif isinstance(chat_template, str):
+        JINJA_CHARS = "{}\n"
+        if (
+            not any(c in chat_template for c in JINJA_CHARS)
+            and not Path(chat_template).exists()
+        ):
+            # Try to find the template in the built-in templates directory
+            from vllm.transformers_utils.chat_templates.registry import (
+                CHAT_TEMPLATES_DIR,
+            )
+
+            builtin_template_path = CHAT_TEMPLATES_DIR / chat_template
+            if not builtin_template_path.exists():
+                raise VLLMValidationError(
+                    f"The supplied chat template string ({chat_template}) "
+                    f"appears path-like, but doesn't exist! "
+                    f"Tried: {chat_template} and {builtin_template_path}",
+                    parameter="chat_template",
+                )
+
+    else:
+        raise VLLMValidationError(
+            f"{type(chat_template)} is not a valid chat template type",
+            parameter="chat_template",
+        )
+
+
+def _load_chat_template(
+    chat_template: Path | str | None,
+    *,
+    is_literal: bool = False,
+) -> str | None:
+    if chat_template is None:
+        return None
+
+    if is_literal:
+        if isinstance(chat_template, Path):
+            raise TypeError(
+                "chat_template is expected to be read directly from its value"
+            )
+
+        return chat_template
+
+    try:
+        with open(chat_template) as f:
+            return f.read()
+    except OSError as e:
+        if isinstance(chat_template, Path):
+            raise
+
+        JINJA_CHARS = "{}\n"
+        if not any(c in chat_template for c in JINJA_CHARS):
+            # Try to load from the built-in templates directory
+            from vllm.transformers_utils.chat_templates.registry import (
+                CHAT_TEMPLATES_DIR,
+            )
+
+            builtin_template_path = CHAT_TEMPLATES_DIR / chat_template
+            try:
+                with open(builtin_template_path) as f:
+                    return f.read()
+            except OSError:
+                msg = (
+                    f"The supplied chat template ({chat_template}) "
+                    f"looks like a file path, but it failed to be opened. "
+                    f"Tried: {chat_template} and {builtin_template_path}. "
+                    f"Reason: {e}"
+                )
+                raise VLLMValidationError(msg, parameter="chat_template") from e
+
+        # If opening a file fails, set chat template to be args to
+        # ensure we decode so our escape are interpreted correctly
+        return _load_chat_template(chat_template, is_literal=True)
+
+
+_cached_load_chat_template = lru_cache(_load_chat_template)
+
+
+def load_chat_template(
+    chat_template: Path | str | None,
+    *,
+    is_literal: bool = False,
+) -> str | None:
+    return _cached_load_chat_template(chat_template, is_literal=is_literal)

vllm/renderers/chat_utils/content_parser.py

@@ -0,0 +1,560 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+import asyncio
+from abc import ABC, abstractmethod
+from collections import defaultdict
+from collections.abc import Awaitable, Callable
+from functools import cached_property, partial
+from typing import TYPE_CHECKING, Any, Final
+
+import torch
+from PIL import Image
+from typing_extensions import override
+
+from vllm import envs
+from vllm.config import ModelConfig
+from vllm.exceptions import VLLMValidationError
+from vllm.multimodal.media import MEDIA_CONNECTOR_REGISTRY, MediaConnector
+from vllm.renderers.embed_utils import (
+    safe_load_prompt_embeds,
+    safe_load_prompt_embeds_async,
+)
+
+from .types import InputAudio, ModalityStr, MultiModalEmbedsPayload
+
+if TYPE_CHECKING:
+    from .item_tracker import AsyncMultiModalItemTracker, MultiModalItemTracker
+
+MODALITY_PLACEHOLDERS_MAP = {
+    "image": "<##IMAGE##>",
+    "audio": "<##AUDIO##>",
+    "video": "<##VIDEO##>",
+    "prompt_embeds": "<##PROMPT_EMBEDS##>",
+}
+
+
+PROMPT_EMBEDS_PLACEHOLDER_TOKEN: Final[str] = "<prompt_embeds>"
+"""The special token used as a placeholder for each embedding
+position during chat template rendering.
+
+Registered as an additional special token when `--enable-prompt-embeds` is set.
+See `_ensure_prompt_embeds_placeholder_token` in `vllm/renderers/hf.py`.
+"""
+
+_ENABLE_PROMPT_EMBEDS_ERROR: Final[str] = (
+    "You must set `--enable-prompt-embeds` to input `prompt_embeds`"
+)
+
+
+class BaseMultiModalContentParser(ABC):
+    def __init__(self) -> None:
+        super().__init__()
+
+        # stores model placeholders list with corresponding
+        # general MM placeholder:
+        # {
+        #   "<##IMAGE##>": ["<image>", "<image>", "<image>"],
+        #   "<##AUDIO##>": ["<audio>", "<audio>"],
+        #   "<##PROMPT_EMBEDS##>": ["<prompt_embeds>", "<prompt_embeds>"]
+        # }
+        self._placeholder_storage: dict[str, list] = defaultdict(list)
+
+    @property
+    @abstractmethod
+    def model_config(self) -> ModelConfig:
+        raise NotImplementedError
+
+    def _add_placeholder(self, modality: ModalityStr, placeholder: str | None):
+        mod_placeholder = MODALITY_PLACEHOLDERS_MAP[modality]
+        if placeholder:
+            self._placeholder_storage[mod_placeholder].append(placeholder)
+
+    def mm_placeholder_storage(self) -> dict[str, list]:
+        return dict(self._placeholder_storage)
+
+    @abstractmethod
+    def parse_image(self, image_url: str | None, uuid: str | None = None) -> None:
+        raise NotImplementedError
+
+    @abstractmethod
+    def parse_image_embeds(
+        self,
+        image_embeds: MultiModalEmbedsPayload | None,
+        uuid: str | None = None,
+    ) -> None:
+        raise NotImplementedError
+
+    @abstractmethod
+    def parse_image_pil(
+        self, image_pil: Image.Image | None, uuid: str | None = None
+    ) -> None:
+        raise NotImplementedError
+
+    @abstractmethod
+    def parse_audio(self, audio_url: str | None, uuid: str | None = None) -> None:
+        raise NotImplementedError
+
+    @abstractmethod
+    def parse_input_audio(
+        self, input_audio: InputAudio | None, uuid: str | None = None
+    ) -> None:
+        raise NotImplementedError
+
+    @abstractmethod
+    def parse_audio_embeds(
+        self,
+        audio_embeds: MultiModalEmbedsPayload | None,
+        uuid: str | None = None,
+    ) -> None:
+        raise NotImplementedError
+
+    @abstractmethod
+    def parse_prompt_embeds(self, data: str) -> None:
+        raise NotImplementedError
+
+    @abstractmethod
+    def parse_video(self, video_url: str | None, uuid: str | None = None) -> None:
+        raise NotImplementedError
+
+    @abstractmethod
+    def parse_video_embeds(
+        self,
+        video_embeds: MultiModalEmbedsPayload | None,
+        uuid: str | None = None,
+    ) -> None:
+        raise NotImplementedError
+
+
+class MultiModalContentParser(BaseMultiModalContentParser):
+    def __init__(
+        self,
+        tracker: "MultiModalItemTracker",
+        mm_processor_kwargs: dict[str, Any] | None = None,
+    ) -> None:
+        super().__init__()
+
+        self._tracker = tracker
+        self._mm_processor_kwargs = mm_processor_kwargs
+
+    @cached_property
+    def _connector(self) -> MediaConnector:
+        # Connector setup may probe VLLM_MEDIA_CACHE. Defer it until a request
+        # actually contains media so text-only parsing never blocks on that I/O.
+        return MEDIA_CONNECTOR_REGISTRY.load(
+            envs.VLLM_MEDIA_CONNECTOR,
+            media_io_kwargs=self._tracker.media_io_kwargs,
+            allowed_local_media_path=self._tracker.allowed_local_media_path,
+            allowed_media_domains=self._tracker.allowed_media_domains,
+            decoded_budget=self._tracker.decoded_budget,
+            download_budget=self._tracker.download_budget,
+        )
+
+    @property
+    def model_config(self) -> ModelConfig:
+        return self._tracker.model_config
+
+    @override
+    def parse_prompt_embeds(self, data: str) -> None:
+        """Decode a base64 prompt embeds tensor and store it in the tracker.
+
+        Emits a single `PROMPT_EMBEDS_PLACEHOLDER_TOKEN` sentinel per
+        content part. The renderer later expands each sentinel to a span of
+        `tensor.shape[0]` placeholder tokens after tokenization.
+        """
+        if not self.model_config.enable_prompt_embeds:
+            raise VLLMValidationError(
+                _ENABLE_PROMPT_EMBEDS_ERROR, parameter="prompt_embeds"
+            )
+
+        tensor = safe_load_prompt_embeds(self.model_config, data.encode())
+        self._tracker.add("prompt_embeds", (tensor, None))
+        self._add_placeholder("prompt_embeds", PROMPT_EMBEDS_PLACEHOLDER_TOKEN)
+
+    def parse_image(self, image_url: str | None, uuid: str | None = None) -> None:
+        image = self._connector.fetch_image(image_url) if image_url else None
+
+        placeholder = self._tracker.add("image", (image, uuid))
+        self._add_placeholder("image", placeholder)
+
+    def parse_image_embeds(
+        self,
+        image_embeds: MultiModalEmbedsPayload | None,
+        uuid: str | None = None,
+    ) -> None:
+        mm_config = self.model_config.get_multimodal_config()
+        if not mm_config.enable_mm_embeds:
+            raise VLLMValidationError(
+                "You must set `--enable-mm-embeds` to input `image_embeds`",
+                parameter="image_embeds",
+            )
+
+        if isinstance(image_embeds, dict):
+            embeds = {
+                k: self._connector.fetch_image_embedding(v) if isinstance(v, str) else v
+                for k, v in image_embeds.items()
+            }
+            placeholder = self._tracker.add("image_embeds", (embeds, uuid))
+
+        if isinstance(image_embeds, str):
+            embedding = self._connector.fetch_image_embedding(image_embeds)
+            placeholder = self._tracker.add("image_embeds", (embedding, uuid))
+
+        if image_embeds is None:
+            placeholder = self._tracker.add("image_embeds", (None, uuid))
+
+        self._add_placeholder("image", placeholder)
+
+    def parse_audio_embeds(
+        self,
+        audio_embeds: MultiModalEmbedsPayload | None,
+        uuid: str | None = None,
+    ) -> None:
+        mm_config = self.model_config.get_multimodal_config()
+        if not mm_config.enable_mm_embeds:
+            raise VLLMValidationError(
+                "You must set `--enable-mm-embeds` to input `audio_embeds`",
+                parameter="audio_embeds",
+            )
+
+        if isinstance(audio_embeds, dict):
+            embeds = {
+                k: self._connector.fetch_audio_embedding(v) if isinstance(v, str) else v
+                for k, v in audio_embeds.items()
+            }
+            placeholder = self._tracker.add("audio_embeds", (embeds, uuid))
+        elif isinstance(audio_embeds, str):
+            embedding = self._connector.fetch_audio_embedding(audio_embeds)
+            placeholder = self._tracker.add("audio_embeds", (embedding, uuid))
+        else:
+            placeholder = self._tracker.add("audio_embeds", (None, uuid))
+
+        self._add_placeholder("audio", placeholder)
+
+    def parse_image_pil(
+        self, image_pil: Image.Image | None, uuid: str | None = None
+    ) -> None:
+        if image_pil is not None:
+            self._tracker.decoded_budget.charge(
+                image_pil.size[0] * image_pil.size[1] * 4
+            )
+        placeholder = self._tracker.add("image", (image_pil, uuid))
+        self._add_placeholder("image", placeholder)
+
+    def parse_audio(self, audio_url: str | None, uuid: str | None = None) -> None:
+        audio = self._connector.fetch_audio(audio_url) if audio_url else None
+
+        placeholder = self._tracker.add("audio", (audio, uuid))
+        self._add_placeholder("audio", placeholder)
+
+    def parse_input_audio(
+        self, input_audio: InputAudio | None, uuid: str | None = None
+    ) -> None:
+        if input_audio:
+            audio_data = input_audio.get("data", "")
+            audio_format = input_audio.get("format", "")
+            if audio_data:
+                audio_url = f"data:audio/{audio_format};base64,{audio_data}"
+            else:
+                # If a UUID is provided, audio data may be empty.
+                audio_url = None
+        else:
+            audio_url = None
+
+        return self.parse_audio(audio_url, uuid)
+
+    def parse_video(self, video_url: str | None, uuid: str | None = None) -> None:
+        video = (
+            self._connector.fetch_video(
+                video_url=video_url,
+                video_processor=self._tracker.video_processor_name,
+            )
+            if video_url
+            else None
+        )
+
+        placeholder = self._tracker.add("video", (video, uuid))
+        self._add_placeholder("video", placeholder)
+
+        # Extract audio from video if use_audio_in_video is True
+        if (
+            video_url
+            and self._mm_processor_kwargs
+            and self._mm_processor_kwargs.get("use_audio_in_video", False)
+        ):
+            audio = self._connector.fetch_audio(video_url) if video_url else None
+            audio_placeholder = self._tracker.add("audio", (audio, uuid))
+            self._add_placeholder("audio", audio_placeholder)
+
+    def parse_video_embeds(
+        self,
+        video_embeds: MultiModalEmbedsPayload | None,
+        uuid: str | None = None,
+    ) -> None:
+        mm_config = self.model_config.get_multimodal_config()
+        if not mm_config.enable_mm_embeds:
+            raise VLLMValidationError(
+                "You must set `--enable-mm-embeds` to input `video_embeds`",
+                parameter="video_embeds",
+            )
+
+        if isinstance(video_embeds, dict):
+            embeds = {
+                k: self._connector.fetch_video_embedding(v) if isinstance(v, str) else v
+                for k, v in video_embeds.items()
+            }
+            placeholder = self._tracker.add("video_embeds", (embeds, uuid))
+        elif isinstance(video_embeds, str):
+            embedding = self._connector.fetch_video_embedding(video_embeds)
+            placeholder = self._tracker.add("video_embeds", (embedding, uuid))
+        else:
+            placeholder = self._tracker.add("video_embeds", (None, uuid))
+
+        self._add_placeholder("video", placeholder)
+
+
+class AsyncMultiModalContentParser(BaseMultiModalContentParser):
+    def __init__(
+        self,
+        tracker: "AsyncMultiModalItemTracker",
+        mm_processor_kwargs: dict[str, Any] | None = None,
+    ) -> None:
+        super().__init__()
+
+        self._tracker = tracker
+        self._mm_processor_kwargs: dict[str, Any] | None = mm_processor_kwargs
+
+    @cached_property
+    def _connector(self) -> MediaConnector:
+        # Connector setup may probe VLLM_MEDIA_CACHE. Defer it until a request
+        # actually contains media so text-only parsing never blocks on that I/O.
+        return MEDIA_CONNECTOR_REGISTRY.load(
+            envs.VLLM_MEDIA_CONNECTOR,
+            media_io_kwargs=self._tracker.media_io_kwargs,
+            allowed_local_media_path=self._tracker.allowed_local_media_path,
+            allowed_media_domains=self._tracker.allowed_media_domains,
+            decoded_budget=self._tracker.decoded_budget,
+            download_budget=self._tracker.download_budget,
+        )
+
+    @property
+    def model_config(self) -> ModelConfig:
+        return self._tracker.model_config
+
+    async def _item_with_uuid_async(self, item: object, uuid: str | None):
+        return item, uuid
+
+    @override
+    def parse_prompt_embeds(self, data: str) -> None:
+        """Schedule async prompt embeds decode and store the coroutine in the tracker.
+
+        Like the sync variant, emits a single sentinel `PROMPT_EMBEDS_PLACEHOLDER_TOKEN`
+        per content part. Unlike the sync variant, the tensor decode is deferred to a
+        thread-pool executor via `safe_load_prompt_embeds_async`.
+        """
+        if not self.model_config.enable_prompt_embeds:
+            raise VLLMValidationError(
+                _ENABLE_PROMPT_EMBEDS_ERROR, parameter="prompt_embeds"
+            )
+
+        self._tracker.add(
+            "prompt_embeds", partial(self._load_prompt_embeds_async, data.encode())
+        )
+        self._add_placeholder("prompt_embeds", PROMPT_EMBEDS_PLACEHOLDER_TOKEN)
+
+    async def _load_prompt_embeds_async(
+        self, data_bytes: bytes
+    ) -> tuple[torch.Tensor, None]:
+        # Second tuple slot fills the tracker's generic `(item, uuid | None)`
+        # contract. prompt_embeds has no UUID concept, so it's always `None`.
+        tensor = await safe_load_prompt_embeds_async(self.model_config, data_bytes)
+        return tensor, None
+
+    async def _image_with_uuid_async(self, image_url: str | None, uuid: str | None):
+        image = (
+            await self._connector.fetch_image_async(image_url) if image_url else None
+        )
+        return image, uuid
+
+    def parse_image(self, image_url: str | None, uuid: str | None = None) -> None:
+        placeholder = self._tracker.add(
+            "image", partial(self._image_with_uuid_async, image_url, uuid)
+        )
+        self._add_placeholder("image", placeholder)
+
+    def parse_image_embeds(
+        self,
+        image_embeds: MultiModalEmbedsPayload | None,
+        uuid: str | None = None,
+    ) -> None:
+        mm_config = self.model_config.get_multimodal_config()
+        if not mm_config.enable_mm_embeds:
+            raise VLLMValidationError(
+                "You must set `--enable-mm-embeds` to input `image_embeds`",
+                parameter="image_embeds",
+            )
+
+        placeholder = self._tracker.add(
+            "image_embeds",
+            partial(self._image_embeds_with_uuid_async, image_embeds, uuid),
+        )
+        self._add_placeholder("image", placeholder)
+
+    async def _image_embeds_with_uuid_async(
+        self,
+        image_embeds: MultiModalEmbedsPayload | None,
+        uuid: str | None,
+    ):
+        if isinstance(image_embeds, dict):
+            embeds = await _load_embeds_dict(
+                image_embeds, self._connector.fetch_image_embedding_async
+            )
+        elif isinstance(image_embeds, str):
+            embeds = await self._connector.fetch_image_embedding_async(image_embeds)
+        else:
+            embeds = None
+        return embeds, uuid
+
+    def parse_audio_embeds(
+        self,
+        audio_embeds: MultiModalEmbedsPayload | None,
+        uuid: str | None = None,
+    ) -> None:
+        mm_config = self.model_config.get_multimodal_config()
+        if not mm_config.enable_mm_embeds:
+            raise VLLMValidationError(
+                "You must set `--enable-mm-embeds` to input `audio_embeds`",
+                parameter="audio_embeds",
+            )
+
+        placeholder = self._tracker.add(
+            "audio_embeds",
+            partial(self._audio_embeds_with_uuid_async, audio_embeds, uuid),
+        )
+        self._add_placeholder("audio", placeholder)
+
+    async def _audio_embeds_with_uuid_async(
+        self,
+        audio_embeds: MultiModalEmbedsPayload | None,
+        uuid: str | None,
+    ):
+        if isinstance(audio_embeds, dict):
+            embeds = await _load_embeds_dict(
+                audio_embeds, self._connector.fetch_audio_embedding_async
+            )
+        elif isinstance(audio_embeds, str):
+            embeds = await self._connector.fetch_audio_embedding_async(audio_embeds)
+        else:
+            embeds = None
+        return embeds, uuid
+
+    def parse_image_pil(
+        self,
+        image_pil: Image.Image | None,
+        uuid: str | None = None,
+    ) -> None:
+        if image_pil is not None:
+            self._tracker.decoded_budget.charge(
+                image_pil.size[0] * image_pil.size[1] * 4
+            )
+        placeholder = self._tracker.add(
+            "image", partial(self._item_with_uuid_async, image_pil, uuid)
+        )
+        self._add_placeholder("image", placeholder)
+
+    async def _audio_with_uuid_async(self, audio_url: str | None, uuid: str | None):
+        audio = (
+            await self._connector.fetch_audio_async(audio_url) if audio_url else None
+        )
+        return audio, uuid
+
+    def parse_audio(self, audio_url: str | None, uuid: str | None = None) -> None:
+        placeholder = self._tracker.add(
+            "audio", partial(self._audio_with_uuid_async, audio_url, uuid)
+        )
+        self._add_placeholder("audio", placeholder)
+
+    def parse_input_audio(
+        self, input_audio: InputAudio | None, uuid: str | None = None
+    ) -> None:
+        if input_audio:
+            audio_data = input_audio.get("data", "")
+            audio_format = input_audio.get("format", "")
+            if audio_data:
+                audio_url = f"data:audio/{audio_format};base64,{audio_data}"
+            else:
+                # If a UUID is provided, audio data may be empty.
+                audio_url = None
+        else:
+            audio_url = None
+
+        return self.parse_audio(audio_url, uuid)
+
+    async def _video_with_uuid_async(self, video_url: str | None, uuid: str | None):
+        video = (
+            await self._connector.fetch_video_async(
+                video_url,
+                video_processor=self._tracker.video_processor_name,
+            )
+            if video_url
+            else None
+        )
+        return video, uuid
+
+    def parse_video(self, video_url: str | None, uuid: str | None = None) -> None:
+        placeholder = self._tracker.add(
+            "video", partial(self._video_with_uuid_async, video_url, uuid)
+        )
+        self._add_placeholder("video", placeholder)
+
+        # Extract audio from video if use_audio_in_video is True
+        if (
+            video_url
+            and self._mm_processor_kwargs
+            and self._mm_processor_kwargs.get("use_audio_in_video", False)
+        ):
+            audio_placeholder = self._tracker.add(
+                "audio", partial(self._audio_with_uuid_async, video_url, uuid)
+            )
+            self._add_placeholder("audio", audio_placeholder)
+
+    def parse_video_embeds(
+        self,
+        video_embeds: MultiModalEmbedsPayload | None,
+        uuid: str | None = None,
+    ) -> None:
+        mm_config = self.model_config.get_multimodal_config()
+        if not mm_config.enable_mm_embeds:
+            raise VLLMValidationError(
+                "You must set `--enable-mm-embeds` to input `video_embeds`",
+                parameter="video_embeds",
+            )
+
+        placeholder = self._tracker.add(
+            "video_embeds",
+            partial(self._video_embeds_with_uuid_async, video_embeds, uuid),
+        )
+        self._add_placeholder("video", placeholder)
+
+    async def _video_embeds_with_uuid_async(
+        self,
+        video_embeds: MultiModalEmbedsPayload | None,
+        uuid: str | None,
+    ):
+        if isinstance(video_embeds, dict):
+            embeds = await _load_embeds_dict(
+                video_embeds, self._connector.fetch_video_embedding_async
+            )
+        elif isinstance(video_embeds, str):
+            embeds = await self._connector.fetch_video_embedding_async(video_embeds)
+        else:
+            embeds = None
+        return embeds, uuid
+
+
+async def _load_embeds_dict(
+    data: dict[str, str | list[int | float]],
+    fetch: Callable[[str], Awaitable["torch.Tensor"]],
+) -> dict[str, Any]:
+    encoded = {key: value for key, value in data.items() if isinstance(value, str)}
+    tensors = await asyncio.gather(*(fetch(value) for value in encoded.values()))
+    return {**data, **dict(zip(encoded, tensors))}

vllm/renderers/chat_utils/item_tracker.py

@@ -0,0 +1,562 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+import asyncio
+from abc import ABC, abstractmethod
+from collections import defaultdict
+from collections.abc import Awaitable, Callable
+from functools import cached_property
+from itertools import accumulate
+from typing import (
+    TYPE_CHECKING,
+    Any,
+    Final,
+    Generic,
+    TypeAlias,
+    TypeVar,
+    cast,
+)
+
+from vllm.config import ModelConfig
+from vllm.exceptions import VLLMValidationError
+from vllm.inputs import MultiModalDataDict, MultiModalUUIDDict
+from vllm.logger import init_logger
+from vllm.model_executor.models import SupportsMultiModal
+from vllm.multimodal import MULTIMODAL_REGISTRY
+from vllm.multimodal.inputs import (
+    MultiModalBatchedField,
+    MultiModalFlatField,
+    MultiModalSharedField,
+    VisionChunk,
+    VisionChunkImage,
+    VisionChunkVideo,
+)
+from vllm.multimodal.media.base import MediaDecodedBudget
+from vllm.multimodal.processing import BaseMultiModalProcessor
+from vllm.transformers_utils.processor import get_video_processor_cls_name
+from vllm.utils import random_uuid
+from vllm.utils.collection_utils import is_list_of, is_list_of_numbers
+from vllm.utils.import_utils import LazyLoader
+
+from ...connections import MediaRequestDownloadBudget
+from .content_parser import (
+    AsyncMultiModalContentParser,
+    BaseMultiModalContentParser,
+    MultiModalContentParser,
+)
+from .types import ModalityStr
+
+_T = TypeVar("_T")
+_AsyncMultiModalItem: TypeAlias = Callable[[], Awaitable[tuple[object, str | None]]]
+
+if TYPE_CHECKING:
+    import torch
+    import transformers
+else:
+    transformers = LazyLoader("transformers", globals(), "transformers")
+    torch = LazyLoader("torch", globals(), "torch")
+
+logger = init_logger(__name__)
+
+
+_REQUIRE_MM_PROCESSOR_ERROR: Final[str] = (
+    "Resolving modality {modality!r} requires a multimodal processor "
+    "but none is available."
+)
+
+
+# Backward compatibility for single item input
+class _BatchedSingleItemField(MultiModalSharedField):
+    pass
+
+
+def _detect_field(
+    tensors: list[torch.Tensor],
+    mm_processor: BaseMultiModalProcessor,
+):
+    first_item = tensors[0]
+    hidden_size = mm_processor.info.ctx.model_config.get_inputs_embeds_size()
+
+    if (
+        len(tensors) == 1
+        and first_item.ndim == 3
+        and first_item.shape[0] == 1
+        and first_item.shape[-1] == hidden_size
+    ):
+        logger.warning(
+            "Batched multi-modal embedding inputs are deprecated for Chat API. "
+            "Please pass a separate content part for each multi-modal item."
+        )
+        return _BatchedSingleItemField(batch_size=1)
+
+    first_shape = first_item.shape
+    if all(t.shape == first_shape for t in tensors):
+        return MultiModalBatchedField()
+
+    size_per_item = [len(tensor) for tensor in tensors]
+    slice_idxs = [0, *accumulate(size_per_item)]
+    slices = [
+        (slice(slice_idxs[i], slice_idxs[i + 1]),) for i in range(len(size_per_item))
+    ]
+    return MultiModalFlatField(slices=slices)
+
+
+def _merge_embeds(
+    data_items: list[dict[str, "torch.Tensor"]],
+    mm_processor: BaseMultiModalProcessor,
+):
+    if not data_items:
+        return {}
+
+    first_keys = set(data_items[0].keys())
+    if any(set(item.keys()) != first_keys for item in data_items[1:]):
+        raise VLLMValidationError(
+            "All dictionaries in the list of embeddings must have the same keys."
+        )
+
+    fields = {
+        key: _detect_field([item[key] for item in data_items], mm_processor)
+        for key in first_keys
+    }
+    data_merged = {
+        key: field._reduce_data([item[key] for item in data_items], pin_memory=False)
+        for key, field in fields.items()
+    }
+
+    try:
+        # TODO: Support per-request mm_processor_kwargs
+        parsed_configs = mm_processor._get_mm_fields_config(
+            transformers.BatchFeature(data_merged),
+            {},
+        )
+        parsed_fields = {key: parsed_configs[key].field for key in first_keys}
+        keys_to_update = [
+            key
+            for key in first_keys
+            if (
+                fields[key] != parsed_fields[key]
+                and not isinstance(fields[key], _BatchedSingleItemField)
+            )
+        ]
+
+        for key in keys_to_update:
+            data_merged[key] = parsed_fields[key]._reduce_data(
+                [item[key] for item in data_items], pin_memory=False
+            )
+    except Exception:
+        logger.exception(
+            "Error when parsing merged embeddings. "
+            "Falling back to auto-detected fields."
+        )
+
+    return data_merged
+
+
+def _parse_metadata_array(key: str, value: list, metadata_fields: set[str]):
+    if key not in metadata_fields:
+        raise VLLMValidationError(f"JSON arrays are only supported for metadata: {key}")
+    if not is_list_of_numbers(value):
+        raise VLLMValidationError(f"Metadata {key} must be a finite numeric array.")
+    dtype = torch.float64 if any(isinstance(v, float) for v in value) else torch.int64
+    try:
+        return torch.tensor(value, dtype=dtype)
+    except (ValueError, TypeError, OverflowError, RuntimeError) as error:
+        raise VLLMValidationError(f"Invalid metadata array: {key}") from error
+
+
+def _get_embeds_data(
+    modality: str,
+    data_items: list[Any],
+    mm_processor: BaseMultiModalProcessor,
+):
+    if len(data_items) == 0:
+        return data_items
+
+    if all(item is None for item in data_items):
+        return data_items
+
+    if is_list_of(data_items, torch.Tensor):
+        embeds_key = f"{modality}_embeds"
+        dict_items = [{embeds_key: item} for item in data_items]
+        return _merge_embeds(dict_items, mm_processor)[embeds_key]
+
+    if is_list_of(data_items, dict):
+        metadata_fields = mm_processor.info.data_parser.placeholder_metadata_fields(
+            modality
+        )
+        data_items = [
+            {
+                key: _parse_metadata_array(key, value, metadata_fields)
+                if isinstance(value, list)
+                else value
+                for key, value in item.items()
+            }
+            for item in data_items
+        ]
+        return _merge_embeds(data_items, mm_processor)
+
+    raise NotImplementedError(type(data_items))
+
+
+class BaseMultiModalItemTracker(ABC, Generic[_T]):
+    """Tracks multi-modal items in a given request and ensures that the number
+    of multi-modal items in a given request does not exceed the configured
+    maximum per prompt.
+    """
+
+    def __init__(
+        self,
+        model_config: ModelConfig,
+        media_io_kwargs: dict[str, dict[str, Any]] | None = None,
+    ):
+        super().__init__()
+
+        self._model_config = model_config
+        self._media_io_kwargs = media_io_kwargs
+
+        self._items_by_modality = defaultdict[str, list[_T]](list)
+        # Track original modality for each vision_chunk item (image or video)
+        self._modality_order = defaultdict[str, list[str]](list)
+        self.decoded_budget = MediaDecodedBudget()
+        self.download_budget = MediaRequestDownloadBudget()
+
+    @cached_property
+    def use_unified_vision_chunk_modality(self) -> bool:
+        """Check if model uses unified vision_chunk modality for images/videos."""
+        return getattr(self._model_config.hf_config, "use_unified_vision_chunk", False)
+
+    @property
+    def model_config(self) -> ModelConfig:
+        return self._model_config
+
+    @cached_property
+    def model_cls(self) -> type[SupportsMultiModal]:
+        from vllm.model_executor.model_loader import get_model_cls
+
+        model_cls = get_model_cls(self.model_config)
+        return cast(type[SupportsMultiModal], model_cls)
+
+    @property
+    def media_io_kwargs(self) -> dict[str, dict[str, Any]] | None:
+        return self._media_io_kwargs or (
+            self._model_config.multimodal_config.media_io_kwargs
+            if self._model_config.multimodal_config
+            else None
+        )
+
+    @property
+    def allowed_local_media_path(self):
+        return self._model_config.allowed_local_media_path
+
+    @property
+    def allowed_media_domains(self):
+        return self._model_config.allowed_media_domains
+
+    @property
+    def mm_registry(self):
+        return MULTIMODAL_REGISTRY
+
+    @cached_property
+    def mm_processor(self):
+        return self.mm_registry.create_processor(self.model_config)
+
+    @property
+    def video_processor_name(self) -> str | None:
+        return get_video_processor_cls_name(self.model_config)
+
+    def add(self, modality: ModalityStr, item: _T) -> str | None:
+        """Add a multi-modal item to the current prompt and returns the
+        placeholder string to use, if any.
+
+        An optional uuid can be added which serves as a unique identifier of the
+        media.
+
+        Note:
+            `prompt_embeds` bypass MM-processor validation because they are
+            pre-computed embeddings that do not go through any HF processor, encoder,
+            or model-specific placeholder logic. The corresponding placeholder string is
+            managed by the parser via `_add_placeholder`, so we return None here.
+
+        """
+        add_info = self._validate_add(modality)
+        if add_info is None:
+            self._items_by_modality["prompt_embeds"].append(item)
+            return None
+
+        input_modality, original_modality, use_vision_chunk, num_items = add_info
+
+        # Track original modality for vision_chunk items
+        if use_vision_chunk:
+            self._items_by_modality[input_modality].append(item)  # type: ignore
+            self._modality_order["vision_chunk"].append(original_modality)
+        else:
+            self._items_by_modality[original_modality].append(item)
+
+        return self.model_cls.get_placeholder_str(modality, num_items)
+
+    def _validate_add(self, modality: ModalityStr) -> tuple[str, str, bool, int] | None:
+        """Validate that one more item of the modality can be tracked."""
+        if modality == "prompt_embeds":
+            return None
+
+        input_modality = modality.replace("_embeds", "")
+        original_modality = modality
+        use_vision_chunk = (
+            self.use_unified_vision_chunk_modality
+            and original_modality in ["video", "image"]
+        )
+
+        # If use_unified_vision_chunk_modality is enabled,
+        # map image/video to vision_chunk
+        if use_vision_chunk:
+            # To avoid validation fail
+            # because models with use_unified_vision_chunk_modality=True
+            # will only accept vision_chunk modality.
+            input_modality = "vision_chunk"
+            num_items = len(self._items_by_modality[input_modality]) + 1
+        else:
+            num_items = len(self._items_by_modality[original_modality]) + 1
+
+        mm_config = self.model_config.multimodal_config
+        if (
+            mm_config is not None
+            and mm_config.enable_mm_embeds
+            and mm_config.get_limit_per_prompt(input_modality) == 0
+            and original_modality.endswith("_embeds")
+        ):
+            # Skip validation: embeddings bypass limit when enable_mm_embeds=True
+            pass
+        else:
+            self.mm_processor.info.validate_num_items(input_modality, num_items)
+
+        return input_modality, original_modality, use_vision_chunk, num_items
+
+    @abstractmethod
+    def create_parser(
+        self, mm_processor_kwargs: dict[str, Any] | None = None
+    ) -> "BaseMultiModalContentParser":
+        raise NotImplementedError
+
+
+def _resolve_vision_chunk_items(
+    vision_chunk_items: list[tuple[object, str | None]],
+    mm_processor: BaseMultiModalProcessor,
+    vision_chunks_modality_order: list[str],
+):
+    # Process vision_chunk items - extract from (data, modality) tuples
+    # and convert to VisionChunk types with proper UUID handling
+    vision_chunks_uuids = [uuid for data, uuid in vision_chunk_items]
+
+    assert len(vision_chunk_items) == len(vision_chunks_modality_order), (
+        f"vision_chunk items ({len(vision_chunk_items)}) and "
+        f"modality_order ({len(vision_chunks_modality_order)}) must have same length"
+    )
+
+    processed_chunks: list[VisionChunk] = []
+    video_idx = 0
+    for inner_modality, (data, uuid) in zip(
+        vision_chunks_modality_order, vision_chunk_items
+    ):
+        if inner_modality == "image":
+            # Cast data to proper type for image
+            # Use .media (PIL.Image) directly to avoid redundant
+            # bytes→PIL conversion in media_processor
+            if hasattr(data, "media"):
+                image_data = data.media  # type: ignore[union-attr]
+                processed_chunks.append(
+                    VisionChunkImage(type="image", image=image_data, uuid=uuid)
+                )
+            else:
+                processed_chunks.append(data)  # type: ignore[arg-type]
+        elif inner_modality == "video":
+            # For video, we may need to split into chunks
+            # if processor supports it
+            # For now, just wrap as a video chunk placeholder
+            if hasattr(mm_processor, "split_video_chunks") and data is not None:
+                try:
+                    video_uuid = uuid or random_uuid()
+                    # video await result is (video_data, video_meta) tuple
+                    if isinstance(data, tuple) and len(data) >= 1:
+                        video_data = data[0]
+                    else:
+                        video_data = data
+                    video_chunks = mm_processor.split_video_chunks(video_data)
+                    for i, vc in enumerate(video_chunks):
+                        processed_chunks.append(
+                            VisionChunkVideo(
+                                type="video_chunk",
+                                video_chunk=vc["video_chunk"],
+                                uuid=f"{video_uuid}-{i}",
+                                video_idx=video_idx,
+                                prompt=vc["prompt"],
+                            )
+                        )
+                    video_idx += 1
+                except Exception as e:
+                    logger.warning("Failed to split video chunks: %s", e)
+                    processed_chunks.append(data)  # type: ignore[arg-type]
+            else:
+                processed_chunks.append(data)  # type: ignore[arg-type]
+    return processed_chunks, vision_chunks_uuids
+
+
+def _resolve_items(
+    items_by_modality: dict[str, list[tuple[object, str | None]]],
+    mm_processor: BaseMultiModalProcessor | None,
+    modality_order: dict[str, list[str]],
+) -> tuple[MultiModalDataDict, MultiModalUUIDDict]:
+    """Materialize the tracker's per-modality items into `mm_data` / `mm_uuids`.
+
+    Note:
+        `mm_processor` is `None` for text-only models (no registered HF
+        processor) whose only modality is `prompt_embeds`. Every other
+        modality requires a processor, enforced by the guard below.
+
+    """
+    if "image" in items_by_modality and "image_embeds" in items_by_modality:
+        raise VLLMValidationError(
+            "Mixing raw image and embedding inputs is not allowed",
+            parameter="image_embeds",
+        )
+    if "audio" in items_by_modality and "audio_embeds" in items_by_modality:
+        raise VLLMValidationError(
+            "Mixing raw audio and embedding inputs is not allowed",
+            parameter="audio_embeds",
+        )
+    if "video" in items_by_modality and "video_embeds" in items_by_modality:
+        raise VLLMValidationError(
+            "Mixing raw video and embedding inputs is not allowed",
+            parameter="video_embeds",
+        )
+    # `prompt_embeds` bypasses HF MM processors. Every other modality requires one.
+    processor_modalities = items_by_modality.keys() - {"prompt_embeds"}
+    if processor_modalities and mm_processor is None:
+        raise RuntimeError(
+            _REQUIRE_MM_PROCESSOR_ERROR.format(modality=processor_modalities)
+        )
+
+    mm_data = {}
+    mm_uuids = {}
+    if "image_embeds" in items_by_modality:
+        assert mm_processor is not None
+        mm_data["image"] = _get_embeds_data(
+            "image",
+            [data for data, uuid in items_by_modality["image_embeds"]],
+            mm_processor,
+        )
+        mm_uuids["image"] = [uuid for data, uuid in items_by_modality["image_embeds"]]
+    if "image" in items_by_modality:
+        mm_data["image"] = [data for data, uuid in items_by_modality["image"]]
+        mm_uuids["image"] = [uuid for data, uuid in items_by_modality["image"]]
+    if "audio_embeds" in items_by_modality:
+        assert mm_processor is not None
+        mm_data["audio"] = _get_embeds_data(
+            "audio",
+            [data for data, uuid in items_by_modality["audio_embeds"]],
+            mm_processor,
+        )
+        mm_uuids["audio"] = [uuid for data, uuid in items_by_modality["audio_embeds"]]
+    if "audio" in items_by_modality:
+        mm_data["audio"] = [data for data, uuid in items_by_modality["audio"]]
+        mm_uuids["audio"] = [uuid for data, uuid in items_by_modality["audio"]]
+    if "video" in items_by_modality:
+        mm_data["video"] = [data for data, uuid in items_by_modality["video"]]
+        mm_uuids["video"] = [uuid for data, uuid in items_by_modality["video"]]
+    if "video_embeds" in items_by_modality:
+        assert mm_processor is not None
+        mm_data["video"] = _get_embeds_data(
+            "video",
+            [data for data, uuid in items_by_modality["video_embeds"]],
+            mm_processor,
+        )
+        mm_uuids["video"] = [uuid for data, uuid in items_by_modality["video_embeds"]]
+    if "vision_chunk" in items_by_modality:
+        assert mm_processor is not None
+        # Process vision_chunk items - extract from (data, modality) tuples
+        # and convert to VisionChunk types with proper UUID handling
+        processed_chunks, vision_chunk_uuids = _resolve_vision_chunk_items(
+            items_by_modality["vision_chunk"],
+            mm_processor,
+            modality_order.get("vision_chunk", []),
+        )
+        mm_data["vision_chunk"] = processed_chunks
+        mm_uuids["vision_chunk"] = vision_chunk_uuids
+    if "prompt_embeds" in items_by_modality:
+        mm_data["prompt_embeds"] = [
+            data for data, _uuid in items_by_modality["prompt_embeds"]
+        ]
+
+    return mm_data, mm_uuids
+
+
+class MultiModalItemTracker(BaseMultiModalItemTracker[tuple[object, str | None]]):
+    def resolve_items(
+        self,
+    ) -> tuple[MultiModalDataDict | None, MultiModalUUIDDict | None]:
+        if not self._items_by_modality:
+            return None, None
+
+        # Text-only models (`is_multimodal_model=False`) with inputs of
+        # modality `prompt_embeds` have no MM processor since `prompt_embeds` are
+        # pre-computed and require no processing, so we pass `None`.
+        mm_processor = (
+            self.mm_processor if self._model_config.is_multimodal_model else None
+        )
+        return _resolve_items(
+            dict(self._items_by_modality),
+            mm_processor,
+            self._modality_order,
+        )
+
+    def create_parser(
+        self, mm_processor_kwargs: dict[str, Any] | None = None
+    ) -> "BaseMultiModalContentParser":
+        return MultiModalContentParser(self, mm_processor_kwargs=mm_processor_kwargs)
+
+
+class AsyncMultiModalItemTracker(BaseMultiModalItemTracker[_AsyncMultiModalItem]):
+    async def resolve_items(
+        self,
+    ) -> tuple[MultiModalDataDict | None, MultiModalUUIDDict | None]:
+        if not self._items_by_modality:
+            return None, None
+
+        # Fetch all modalities together. Each tracked item is already an
+        # independent awaitable, and the async connector offloads blocking
+        # decode work, so waiting for one modality before starting the next
+        # needlessly adds their latency.
+        # Keep the original group and item order when rebuilding the result.
+        item_groups = list(self._items_by_modality.items())
+        items = [item for _, group in item_groups for item in group]
+        results = await asyncio.gather(
+            *(item() for item in items), return_exceptions=True
+        )
+        for result in results:
+            if isinstance(result, BaseException):
+                # Gathering with return_exceptions=True lets every task finish
+                # (or itself fail) before we raise, instead of abandoning
+                # still-in-flight fetches (real network/thread-pool work) the
+                # moment the first one fails.
+                raise result
+
+        resolved_items_by_modality: dict[str, list[Any]] = {}
+        result_idx = 0
+        for modality, group in item_groups:
+            next_result_idx = result_idx + len(group)
+            resolved_items_by_modality[modality] = results[result_idx:next_result_idx]
+            result_idx = next_result_idx
+
+        mm_processor = (
+            self.mm_processor if self._model_config.is_multimodal_model else None
+        )
+        return _resolve_items(
+            resolved_items_by_modality,
+            mm_processor,
+            self._modality_order,
+        )
+
+    def create_parser(
+        self, mm_processor_kwargs: dict[str, Any] | None = None
+    ) -> "BaseMultiModalContentParser":
+        return AsyncMultiModalContentParser(
+            self, mm_processor_kwargs=mm_processor_kwargs
+        )

vllm/renderers/chat_utils/tool_call.py

@@ -0,0 +1,32 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+from vllm.config import ModelConfig
+from vllm.utils import random_uuid
+
+_KIMI_MODEL_TYPES = ("kimi_k2", "kimi_k25", "kimi_k3")
+
+
+def get_tool_call_id_type(model_config: ModelConfig) -> str:
+    """Return the tool-call ID type for a given model configuration."""
+    hf_overrides = getattr(model_config, "hf_overrides", None)
+    hf_config = getattr(model_config, "hf_config", None)
+    hf_text_config = getattr(model_config, "hf_text_config", None)
+    model_types = (
+        getattr(hf_config, "model_type", None),
+        getattr(hf_text_config, "model_type", None),
+    )
+    if any(model_type in _KIMI_MODEL_TYPES for model_type in model_types) or (
+        isinstance(hf_overrides, dict)
+        and hf_overrides.get("model_type") in _KIMI_MODEL_TYPES
+    ):
+        return "kimi_k2"
+    return "random"
+
+
+def make_tool_call_id(id_type: str = "random", func_name=None, idx=None):
+    if id_type == "kimi_k2":
+        return f"functions.{func_name}:{idx}"
+    else:
+        # by default return random
+        return f"chatcmpl-tool-{random_uuid()}"

vllm/renderers/chat_utils/types.py

@@ -0,0 +1,367 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+from typing import Literal, Required, TypeAlias
+
+from openai.types.chat import (
+    ChatCompletionContentPartImageParam,
+    ChatCompletionContentPartInputAudioParam,
+    ChatCompletionContentPartRefusalParam,
+    ChatCompletionContentPartTextParam,
+    ChatCompletionFunctionToolParam,
+    ChatCompletionMessageToolCallParam,
+)
+from openai.types.chat import (
+    ChatCompletionContentPartParam as OpenAIChatCompletionContentPartParam,
+)
+from openai.types.chat import (
+    ChatCompletionMessageParam as OpenAIChatCompletionMessageParam,
+)
+from openai.types.chat.chat_completion_content_part_image_param import ImageURL
+from openai.types.chat.chat_completion_content_part_input_audio_param import InputAudio
+from openai_harmony import Message as OpenAIHarmonyMessage
+from PIL import Image
+from pydantic import BaseModel, ConfigDict
+
+# pydantic needs the TypedDict from typing_extensions
+from typing_extensions import TypedDict
+
+__all__ = [
+    "ChatCompletionContentPartAudioEmbedsParam",
+    "ChatCompletionContentPartAudioParam",
+    "ChatCompletionContentPartImageEmbedsParam",
+    "ChatCompletionContentPartImageParam",
+    "ChatCompletionContentPartParam",
+    "ChatCompletionContentPartPromptEmbedsParam",
+    "ChatCompletionContentPartTextParam",
+    "ChatCompletionContentPartVideoEmbedsParam",
+    "ChatCompletionContentPartVideoParam",
+    "ChatCompletionMessageParam",
+    "ChatTemplateContentFormat",
+    "ChatTemplateContentFormatOption",
+    "ConversationMessage",
+    "CustomChatCompletionContentPILImageParam",
+    "CustomChatCompletionContentSimpleAudioParam",
+    "CustomChatCompletionContentSimpleImageParam",
+    "CustomChatCompletionContentSimpleVideoParam",
+    "CustomChatCompletionContentToolReferenceParam",
+    "CustomChatCompletionMessageParam",
+    "CustomThinkCompletionContentParam",
+    "ImageURL",
+    "InputAudio",
+    "ModalityStr",
+    "MultiModalEmbedsPayload",
+    "PILImage",
+]
+
+
+class AudioURL(TypedDict, total=False):
+    url: Required[str]
+    """
+    Either a URL of the audio or a data URL with base64 encoded audio data.
+    """
+
+
+class ChatCompletionContentPartAudioParam(TypedDict, total=False):
+    audio_url: Required[AudioURL]
+
+    type: Required[Literal["audio_url"]]
+    """The type of the content part."""
+
+
+MultiModalEmbedsPayload: TypeAlias = str | dict[str, str | list[int | float]]
+
+
+class ChatCompletionContentPartImageEmbedsParam(TypedDict, total=False):
+    image_embeds: MultiModalEmbedsPayload | None
+    """
+    The image embeddings. It can be either:
+    - A single base64 string.
+    - A dictionary of base64 tensors or numeric JSON metadata arrays.
+    """
+    type: Required[Literal["image_embeds"]]
+    """The type of the content part."""
+    uuid: str | None
+    """
+    User-provided UUID of a media. User must guarantee that it is properly
+    generated and unique for different medias.
+    """
+
+
+class ChatCompletionContentPartAudioEmbedsParam(TypedDict, total=False):
+    audio_embeds: MultiModalEmbedsPayload | None
+    """
+    The audio embeddings. It can be either:
+    - A single base64 string representing a serialized torch tensor.
+    - A dictionary of base64 tensors or numeric JSON metadata arrays.
+    """
+    type: Required[Literal["audio_embeds"]]
+    """The type of the content part."""
+    uuid: str | None
+    """
+    User-provided UUID of a media. User must guarantee that it is properly
+    generated and unique for different medias.
+    """
+
+
+class ChatCompletionContentPartVideoEmbedsParam(TypedDict, total=False):
+    video_embeds: MultiModalEmbedsPayload | None
+    """
+    The video embeddings. It can be either:
+    - A single base64 string representing a serialized torch tensor.
+    - A dictionary of base64 tensors or numeric JSON metadata arrays.
+    """
+    type: Required[Literal["video_embeds"]]
+    """The type of the content part."""
+    uuid: str | None
+    """
+    User-provided UUID of a media. User must guarantee that it is properly
+    generated and unique for different medias.
+    """
+
+
+class ChatCompletionContentPartPromptEmbedsParam(TypedDict, total=False):
+    data: Required[str]
+    """
+    Base64-encoded bytes of a serialized `torch.Tensor` of shape
+    `(num_tokens, hidden_size)`. The tensor's `dtype` and `hidden_size` must
+    match the model's input embedding layer.
+    """
+    type: Required[Literal["prompt_embeds"]]
+    """The type of the content part."""
+
+
+class VideoURL(TypedDict, total=False):
+    url: Required[str]
+    """
+    Either a URL of the video or a data URL with base64 encoded video data.
+    """
+
+
+class ChatCompletionContentPartVideoParam(TypedDict, total=False):
+    video_url: Required[VideoURL]
+
+    type: Required[Literal["video_url"]]
+    """The type of the content part."""
+
+
+class PILImage(BaseModel):
+    """A PIL.Image.Image object."""
+
+    image_pil: Image.Image
+    model_config = ConfigDict(arbitrary_types_allowed=True)
+
+
+class CustomChatCompletionContentPILImageParam(TypedDict, total=False):
+    """A simpler version of the param that only accepts a PIL image.
+
+    Example:
+    {
+        "image_pil": ImageAsset('cherry_blossom').pil_image
+    }
+
+    """
+
+    image_pil: PILImage | None
+    uuid: str | None
+    """
+    User-provided UUID of a media. User must guarantee that it is properly
+    generated and unique for different medias.
+    """
+
+
+class CustomChatCompletionContentSimpleImageParam(TypedDict, total=False):
+    """A simpler version of the param that only accepts a plain image_url.
+    This is supported by OpenAI API, although it is not documented.
+
+    Example:
+    {
+        "image_url": "https://example.com/image.jpg"
+    }
+
+    """
+
+    image_url: str | None
+    uuid: str | None
+    """
+    User-provided UUID of a media. User must guarantee that it is properly
+    generated and unique for different medias.
+    """
+
+
+class CustomChatCompletionContentSimpleAudioParam(TypedDict, total=False):
+    """A simpler version of the param that only accepts a plain audio_url.
+
+    Example:
+    {
+        "audio_url": "https://example.com/audio.mp3"
+    }
+
+    """
+
+    audio_url: str | None
+
+
+class CustomChatCompletionContentSimpleVideoParam(TypedDict, total=False):
+    """A simpler version of the param that only accepts a plain audio_url.
+
+    Example:
+    {
+        "video_url": "https://example.com/video.mp4"
+    }
+
+    """
+
+    video_url: str | None
+    uuid: str | None
+    """
+    User-provided UUID of a media. User must guarantee that it is properly
+    generated and unique for different medias.
+    """
+
+
+class CustomThinkCompletionContentParam(TypedDict, total=False):
+    """A Think Completion Content Param that accepts a plain text and a boolean.
+
+    Example:
+    {
+        "thinking": "I am thinking about the answer",
+        "closed": True,
+        "type": "thinking"
+    }
+
+    """
+
+    thinking: Required[str]
+    """The thinking content."""
+
+    closed: bool
+    """Whether the thinking is closed."""
+
+    type: Required[Literal["thinking"]]
+    """The thinking type."""
+
+
+class CustomChatCompletionContentToolReferenceParam(TypedDict, total=False):
+    """A tool reference content param that only accepts a plain tool name.
+
+    Example:
+    {
+        "name": "get_weather",
+        "type": "tool_reference"
+    }
+
+    """
+
+    name: str
+    """The name of the tool being referenced."""
+
+    type: Literal["tool_reference"]
+    """The content type."""
+
+
+ChatCompletionContentPartParam: TypeAlias = (
+    OpenAIChatCompletionContentPartParam
+    | ChatCompletionContentPartAudioParam
+    | ChatCompletionContentPartInputAudioParam
+    | ChatCompletionContentPartVideoParam
+    | ChatCompletionContentPartRefusalParam
+    | CustomChatCompletionContentPILImageParam
+    | CustomChatCompletionContentSimpleImageParam
+    | ChatCompletionContentPartImageEmbedsParam
+    | ChatCompletionContentPartAudioEmbedsParam
+    | ChatCompletionContentPartVideoEmbedsParam
+    | ChatCompletionContentPartPromptEmbedsParam
+    | CustomChatCompletionContentSimpleAudioParam
+    | CustomChatCompletionContentSimpleVideoParam
+    | CustomChatCompletionContentToolReferenceParam
+    | str
+    | CustomThinkCompletionContentParam
+)
+
+
+class CustomChatCompletionMessageParam(TypedDict, total=False):
+    """Enables custom roles in the Chat Completion API."""
+
+    role: Required[str]
+    """The role of the message's author."""
+
+    content: str | list[ChatCompletionContentPartParam]
+    """The contents of the message."""
+
+    name: str
+    """An optional name for the participant.
+
+    Provides the model information to differentiate between participants of the
+    same role.
+    """
+
+    tool_call_id: str | None
+    """Tool call that this message is responding to."""
+
+    tool_calls: list[ChatCompletionMessageToolCallParam] | None
+    """The tool calls generated by the model, such as function calls."""
+
+    reasoning: str | None
+    """The reasoning content for interleaved thinking."""
+
+    tools: list[ChatCompletionFunctionToolParam] | None
+    """The tools for developer role."""
+
+    task: str | None
+    """Model-specific task marker. Currently passed through for DeepSeek V4."""
+
+
+ChatCompletionMessageParam: TypeAlias = (
+    OpenAIChatCompletionMessageParam
+    | CustomChatCompletionMessageParam
+    | OpenAIHarmonyMessage
+)
+
+
+# TODO: Make fields ReadOnly once mypy supports it
+class ConversationMessage(TypedDict, total=False):
+    role: Required[str]
+    """The role of the message's author."""
+
+    content: str | None | list[dict[str, str]]
+    """The contents of the message"""
+
+    tool_call_id: str | None
+    """Tool call that this message is responding to."""
+
+    name: str | None
+    """The name of the function to call"""
+
+    tool_calls: list[ChatCompletionMessageToolCallParam] | None
+    """The tool calls generated by the model, such as function calls."""
+
+    reasoning: str | None
+    """The reasoning content for interleaved thinking."""
+
+    reasoning_content: str | None
+    """Deprecated: The reasoning content for interleaved thinking."""
+
+    tools: list[ChatCompletionFunctionToolParam] | None
+    """The tools for developer role."""
+
+    task: str | None
+    """Model-specific task marker. Currently passed through for DeepSeek V4."""
+
+
+# Passed in by user
+ChatTemplateContentFormatOption = Literal["auto", "string", "openai"]
+
+# After resolving "auto"
+ChatTemplateContentFormat = Literal["string", "openai"]
+
+
+ModalityStr = Literal[
+    "image",
+    "audio",
+    "video",
+    "image_embeds",
+    "audio_embeds",
+    "video_embeds",
+    "vision_chunk",
+    "prompt_embeds",
+]

vllm/renderers/hf.py

@@ -22,7 +22,7 @@ import torch
 from typing_extensions import override
 
 from vllm import envs
-from vllm.exceptions import VLLMValidationError
+from vllm.exceptions import ChatTemplateResolutionError, VLLMValidationError
 from vllm.inputs import EmbedsPrompt
 from vllm.inputs.engine import MultiModalInput
 from vllm.logger import init_logger
@@ -41,7 +41,6 @@ from vllm.multimodal.processing.processor import (
 )
 from vllm.renderers.chat_utils import (
     PROMPT_EMBEDS_PLACEHOLDER_TOKEN,
-    ChatTemplateResolutionError,
     load_chat_template,
     parse_chat_messages,
     parse_chat_messages_async,