# SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import json import types from collections import Counter from collections.abc import Callable, Iterable from functools import partial from typing import Any, Final, TypeAlias, Union, cast, get_args, get_origin from openai.types.chat import ( ChatCompletionAssistantMessageParam, ChatCompletionContentPartImageParam, ChatCompletionContentPartInputAudioParam, ChatCompletionContentPartRefusalParam, ChatCompletionToolMessageParam, ) from openai.types.responses import ResponseInputImageParam from PIL import Image from pydantic import TypeAdapter from vllm.config import ModelConfig from vllm.exceptions import VLLMValidationError from vllm.inputs import MultiModalDataDict, MultiModalUUIDDict from vllm.logger import init_logger from .content_parser import ( MODALITY_PLACEHOLDERS_MAP, PROMPT_EMBEDS_PLACEHOLDER_TOKEN, BaseMultiModalContentParser, ) from .item_tracker import ( AsyncMultiModalItemTracker, BaseMultiModalItemTracker, MultiModalItemTracker, ) from .types import ( ChatCompletionContentPartAudioEmbedsParam, ChatCompletionContentPartAudioParam, ChatCompletionContentPartImageEmbedsParam, ChatCompletionContentPartParam, ChatCompletionContentPartPromptEmbedsParam, ChatCompletionContentPartTextParam, ChatCompletionContentPartVideoEmbedsParam, ChatCompletionContentPartVideoParam, ChatCompletionMessageParam, ChatTemplateContentFormat, ConversationMessage, CustomChatCompletionContentPILImageParam, CustomChatCompletionContentSimpleAudioParam, CustomChatCompletionContentSimpleImageParam, CustomChatCompletionContentSimpleVideoParam, CustomChatCompletionContentToolReferenceParam, CustomThinkCompletionContentParam, InputAudio, MultiModalEmbedsPayload, PILImage, ) logger = init_logger(__name__) _PROMPT_EMBEDS_MISSING_DATA_ERROR: Final[str] = ( "prompt_embeds content part requires a non-empty `data` field " "with base64-encoded tensor bytes." ) _RESERVED_PLACEHOLDER_IN_TEXT_ERROR: Final[str] = ( "Text content may not contain the reserved placeholder {token!r}. " "This placeholder is used internally to mark `prompt_embeds` splice " "positions in the tokenized prompt." ) # No need to validate using Pydantic again _TextParser = partial(cast, ChatCompletionContentPartTextParam) _ImageEmbedsParser = partial(cast, ChatCompletionContentPartImageEmbedsParam) _AudioEmbedsParser = partial(cast, ChatCompletionContentPartAudioEmbedsParam) _VideoEmbedsParser = partial(cast, ChatCompletionContentPartVideoEmbedsParam) _PromptEmbedsParser = partial(cast, ChatCompletionContentPartPromptEmbedsParam) _InputAudioParser = partial(cast, ChatCompletionContentPartInputAudioParam) _RefusalParser = partial(cast, ChatCompletionContentPartRefusalParam) _PILImageParser = partial(cast, CustomChatCompletionContentPILImageParam) _ThinkParser = partial(cast, CustomThinkCompletionContentParam) _AssistantParser = partial(cast, ChatCompletionAssistantMessageParam) _ToolParser = partial(cast, ChatCompletionToolMessageParam) # Need to validate url objects _ImageParser = TypeAdapter(ChatCompletionContentPartImageParam).validate_python _AudioParser = TypeAdapter(ChatCompletionContentPartAudioParam).validate_python _VideoParser = TypeAdapter(ChatCompletionContentPartVideoParam).validate_python _ResponsesInputImageParser = TypeAdapter(ResponseInputImageParam).validate_python ContentPart: TypeAlias = ( MultiModalEmbedsPayload | dict[str, str] | InputAudio | PILImage ) # Define a mapping from part types to their corresponding parsing functions. MM_PARSER_MAP: dict[ str, Callable[[ChatCompletionContentPartParam], ContentPart], ] = { "text": lambda part: _TextParser(part).get("text", None), "thinking": lambda part: _ThinkParser(part).get("thinking", None), "input_text": lambda part: _TextParser(part).get("text", None), "output_text": lambda part: _TextParser(part).get("text", None), "input_image": lambda part: _ResponsesInputImageParser(part).get("image_url", None), "image_url": lambda part: _ImageParser(part).get("image_url", {}).get("url", None), "image_embeds": lambda part: _ImageEmbedsParser(part).get("image_embeds", None), "audio_embeds": lambda part: _AudioEmbedsParser(part).get("audio_embeds", None), "video_embeds": lambda part: _VideoEmbedsParser(part).get("video_embeds", None), "prompt_embeds": lambda part: _PromptEmbedsParser(part).get("data", None), "image_pil": lambda part: _PILImageParser(part).get("image_pil", None), "audio_url": lambda part: _AudioParser(part).get("audio_url", {}).get("url", None), "input_audio": lambda part: _InputAudioParser(part).get("input_audio", None), "refusal": lambda part: _RefusalParser(part).get("refusal", None), "video_url": lambda part: _VideoParser(part).get("video_url", {}).get("url", None), "tool_reference": lambda part: cast( CustomChatCompletionContentToolReferenceParam, part ).get("name", None), } def _get_interleaved_text_prompt( placeholder_storage: dict[str, list], texts: list[str] ) -> str: for idx, elem in enumerate(texts): if elem in placeholder_storage: slots = placeholder_storage[elem] if not slots: raise VLLMValidationError( f"Found more '{elem}' placeholders in input prompt than " "actual multimodal data items." ) texts[idx] = slots.pop(0) return "\n".join(texts) # TODO: Let user specify how to insert multimodal tokens into prompt # (similar to chat template) def get_full_multimodal_text_prompt( placeholder_storage: dict[str, list], texts: list[str], interleave_strings: bool, multimodal_content_part_separator: str = "\n", ) -> str: """Combine multimodal prompts for a multimodal language model.""" # flatten storage to make it looks like # { # "<|image|>": 2, # "<|audio|>": 1 # } placeholder_counts = Counter( [v for elem in placeholder_storage.values() for v in elem] ) if interleave_strings: text_prompt = _get_interleaved_text_prompt(placeholder_storage, texts) else: text_prompt = "\n".join(texts) # Pass interleaved text further in case the user used image placeholders # himself, but forgot to disable the 'interleave_strings' flag # Look through the text prompt to check for missing placeholders missing_placeholders: list[str] = [] for placeholder in placeholder_counts: # For any existing placeholder in the text prompt, we leave it as is placeholder_counts[placeholder] -= text_prompt.count(placeholder) if placeholder_counts[placeholder] < 0: logger.error( "Placeholder count is negative! " "Ensure that the 'interleave_strings' flag is disabled " "(current value: %s) " "when manually placing image placeholders.", interleave_strings, ) logger.debug("Input prompt: %s", text_prompt) raise VLLMValidationError( f"Found more '{placeholder}' placeholders in input prompt than " "actual multimodal data items.", parameter="messages", ) missing_placeholders.extend([placeholder] * placeholder_counts[placeholder]) # NOTE: Default behaviour: we always add missing placeholders # at the front of the prompt, if interleave_strings=False if text_prompt: return multimodal_content_part_separator.join( missing_placeholders + [text_prompt] ) else: return multimodal_content_part_separator.join(missing_placeholders) def _collect_known_content_part_fields() -> frozenset[str]: fields: set[str] = set() stack: list[Any] = [ChatCompletionContentPartParam] while stack: node = stack.pop() if get_origin(node) in (Union, types.UnionType): stack.extend(get_args(node)) elif hasattr(node, "__required_keys__"): fields |= node.__required_keys__ | node.__optional_keys__ return frozenset(fields) _KNOWN_CONTENT_PART_FIELDS = _collect_known_content_part_fields() def _collect_extra_fields(part: dict[str, Any]) -> dict[str, Any]: return {k: v for k, v in part.items() if k not in _KNOWN_CONTENT_PART_FIELDS} def parse_chat_message_content_mm_part( part: ChatCompletionContentPartParam, ) -> tuple[str, ContentPart]: """Parses a given multi-modal content part based on its type. Args: part: A dict containing the content part, with a potential 'type' field. Returns: A tuple (part_type, content) where: - part_type: Type of the part (e.g., 'text', 'image_url'). - content: Parsed content (e.g., text, image URL). Raises: ValueError: If the 'type' field is missing and no direct URL is found. """ assert isinstance( part, dict ) # This is needed to avoid mypy errors: part.get() from str part_type = part.get("type", None) uuid = part.get("uuid", None) if isinstance(part_type, str) and part_type in MM_PARSER_MAP and uuid is None: # noqa: E501 content = MM_PARSER_MAP[part_type](part) # Special case for 'image_url.detail' # We only support 'auto', which is the default if part_type == "image_url" and part.get("detail", "auto") != "auto": logger.warning( "'image_url.detail' is currently not supported and will be ignored." ) return part_type, content # Handle missing 'type' but provided direct URL fields. # 'type' is required field by pydantic if part_type is None or uuid is not None: if "image_url" in part: image_params = cast(CustomChatCompletionContentSimpleImageParam, part) image_url = image_params.get("image_url", None) if isinstance(image_url, dict): # Can potentially happen if user provides a uuid # with url as a dict of {"url": url} image_url = image_url.get("url", None) return "image_url", image_url if "image_pil" in part: # "image_pil" could be None if UUID is provided. image_params = cast( # type: ignore CustomChatCompletionContentPILImageParam, part ) image_pil = image_params.get("image_pil", None) return "image_pil", image_pil if "image_embeds" in part: # "image_embeds" could be None if UUID is provided. image_params = cast( # type: ignore ChatCompletionContentPartImageEmbedsParam, part ) image_embeds = image_params.get("image_embeds", None) return "image_embeds", image_embeds if "audio_embeds" in part: # "audio_embeds" could be None if UUID is provided. audio_params = cast( # type: ignore[assignment] ChatCompletionContentPartAudioEmbedsParam, part ) audio_embeds = audio_params.get("audio_embeds", None) return "audio_embeds", audio_embeds if "video_embeds" in part: # "video_embeds" could be None if UUID is provided. video_embeds_params = cast( # type: ignore[assignment] ChatCompletionContentPartVideoEmbedsParam, part ) video_embeds = video_embeds_params.get("video_embeds", None) return "video_embeds", video_embeds if "prompt_embeds" in part: prompt_embeds_params = cast( # type: ignore[assignment] ChatCompletionContentPartPromptEmbedsParam, part ) return "prompt_embeds", prompt_embeds_params.get("data", None) if "audio_url" in part: audio_params = cast( # type: ignore[assignment] CustomChatCompletionContentSimpleAudioParam, part ) audio_url = audio_params.get("audio_url", None) if isinstance(audio_url, dict): # Can potentially happen if user provides a uuid # with url as a dict of {"url": url} audio_url = audio_url.get("url", None) return "audio_url", audio_url if "input_audio" in part: input_audio_params = _InputAudioParser(part).get("input_audio", None) return "input_audio", input_audio_params if "video_url" in part: video_params = cast(CustomChatCompletionContentSimpleVideoParam, part) video_url = video_params.get("video_url", None) if isinstance(video_url, dict): # Can potentially happen if user provides a uuid # with url as a dict of {"url": url} video_url = video_url.get("url", None) return "video_url", video_url if "tool_reference" in part: tool_reference_params = cast( CustomChatCompletionContentToolReferenceParam, part ) tool_reference = tool_reference_params.get("name", None) return "tool_reference", tool_reference # Raise an error if no 'type' or direct URL is found. raise VLLMValidationError( "Missing 'type' field in multimodal part.", parameter="type" ) if not isinstance(part_type, str): raise VLLMValidationError( "Invalid 'type' field in multimodal part.", parameter="type" ) return part_type, "unknown part_type content" PART_TYPES_TO_SKIP_NONE_CONTENT = ( "text", "refusal", ) # Content part types parsed as text rather than multimodal data. TEXT_PART_TYPES = frozenset( {"text", "input_text", "output_text", "refusal", "thinking"} ) # Content part types that carry no multimodal data. _TEXT_CONTENT_PART_TYPES = TEXT_PART_TYPES | {"tool_reference"} # Keys that mark a content part as multimodal, whatever its ``type``. _MEDIA_CONTENT_PART_KEYS = frozenset(MM_PARSER_MAP) - _TEXT_CONTENT_PART_TYPES def has_non_text_content(messages: Any) -> bool: """Whether any message in ``messages`` has a non-text content part. Only list content is inspected, so validated chat content that is a one-shot iterator is never consumed. """ if not isinstance(messages, list): return False for msg in messages: content = msg.get("content") if isinstance(msg, dict) else None if not isinstance(content, list): continue for part in content: if isinstance(part, dict) and ( any(key in part for key in _MEDIA_CONTENT_PART_KEYS) or not isinstance(part_type := part.get("type", "text"), str) or part_type not in _TEXT_CONTENT_PART_TYPES ): return True return False def parse_chat_message_content_parts( role: str, parts: Iterable[ChatCompletionContentPartParam], mm_tracker: BaseMultiModalItemTracker, *, wrap_dicts: bool, interleave_strings: bool, mm_processor_kwargs: dict[str, Any] | None = None, multimodal_content_part_separator="\n", ) -> list[ConversationMessage]: content = list[ContentPart]() mm_parser = mm_tracker.create_parser(mm_processor_kwargs=mm_processor_kwargs) for part in parts: parse_res = parse_chat_message_content_part( part, mm_parser, wrap_dicts=wrap_dicts, interleave_strings=interleave_strings, ) if parse_res: content.append(parse_res) if wrap_dicts: # Parsing wraps images and texts as interleaved dictionaries return [ConversationMessage(role=role, content=content)] # type: ignore texts = cast(list[str], content) mm_placeholder_storage = mm_parser.mm_placeholder_storage() if mm_placeholder_storage: text_prompt = get_full_multimodal_text_prompt( mm_placeholder_storage, texts, interleave_strings, multimodal_content_part_separator=multimodal_content_part_separator, ) else: text_prompt = "\n".join(texts) return [ConversationMessage(role=role, content=text_prompt)] def _reject_reserved_placeholder_in_text(text: str, model_config: ModelConfig) -> None: """Reject user-supplied text parts that contains the reserved `prompt_embeds` placeholder sentinel. When the server accepts `prompt_embeds`, the placeholder token is registered as a single unsplittable special token on the tokenizer. Any user text that happens to contain the literal sequence would tokenize to the same ID and be mistaken for a splice point by the renderer, letting a caller move or inject splice positions via plain text content. """ if model_config.enable_prompt_embeds and PROMPT_EMBEDS_PLACEHOLDER_TOKEN in text: raise VLLMValidationError( _RESERVED_PLACEHOLDER_IN_TEXT_ERROR.format( token=PROMPT_EMBEDS_PLACEHOLDER_TOKEN ), parameter="messages", ) def parse_chat_message_content_part( part: ChatCompletionContentPartParam, mm_parser: BaseMultiModalContentParser, *, wrap_dicts: bool, interleave_strings: bool, ) -> ContentPart | None: """Parses a single part of a conversation. If wrap_dicts is True, structured dictionary pieces for texts and images will be wrapped in dictionaries, i.e., {"type": "text", "text", ...} and {"type": "image"}, respectively. Otherwise multimodal data will be handled by mm_parser, and texts will be returned as strings to be joined with multimodal placeholders. """ if isinstance(part, str): # Handle plain text parts _reject_reserved_placeholder_in_text(part, mm_parser.model_config) if wrap_dicts: return {"type": "text", "text": part} return part # Handle structured dictionary parts part_type, content = parse_chat_message_content_mm_part(part) # if part_type is text/refusal/image_url/audio_url/video_url/input_audio but # content is None, log a warning and skip if part_type in PART_TYPES_TO_SKIP_NONE_CONTENT and content is None: logger.warning( "Skipping multimodal part '%s' (type: '%s') " "with empty / unparsable content.", part, part_type, ) return None if part_type in TEXT_PART_TYPES: str_content = cast(str, content) _reject_reserved_placeholder_in_text(str_content, mm_parser.model_config) if wrap_dicts: result: dict[str, Any] = {"type": "text", "text": str_content} result.update(_collect_extra_fields(cast(dict[str, Any], part))) return result else: return str_content # For media items, if a user has provided one, use it. Otherwise, insert # a placeholder empty uuid. uuid = part.get("uuid", None) if uuid is not None: uuid = str(uuid) modality = None if part_type == "image_pil": image_content = cast(Image.Image, content) if content is not None else None mm_parser.parse_image_pil(image_content, uuid) modality = "image" elif part_type in ("image_url", "input_image"): str_content = cast(str, content) mm_parser.parse_image(str_content, uuid) modality = "image" elif part_type == "image_embeds": content = ( cast(MultiModalEmbedsPayload, content) if content is not None else None ) mm_parser.parse_image_embeds(content, uuid) modality = "image" elif part_type == "audio_embeds": content = ( cast(MultiModalEmbedsPayload, content) if content is not None else None ) mm_parser.parse_audio_embeds(content, uuid) modality = "audio" elif part_type == "video_embeds": content = ( cast(MultiModalEmbedsPayload, content) if content is not None else None ) mm_parser.parse_video_embeds(content, uuid) modality = "video" elif part_type == "prompt_embeds": if not content: raise VLLMValidationError( _PROMPT_EMBEDS_MISSING_DATA_ERROR, parameter="prompt_embeds" ) mm_parser.parse_prompt_embeds(cast(str, content)) modality = "prompt_embeds" elif part_type == "audio_url": str_content = cast(str, content) mm_parser.parse_audio(str_content, uuid) modality = "audio" elif part_type == "input_audio": dict_content = cast(InputAudio, content) mm_parser.parse_input_audio(dict_content, uuid) modality = "audio" elif part_type == "video_url": str_content = cast(str, content) mm_parser.parse_video(str_content, uuid) modality = "video" elif part_type == "tool_reference": # Tool references are not multimodal data — they reference deferred # tools and are passed through as-is for the chat template to expand. if wrap_dicts: return {"type": "tool_reference", "name": cast(str, content)} return cast(str, content) else: supported = sorted(MM_PARSER_MAP.keys() | set(PART_TYPES_TO_SKIP_NONE_CONTENT)) raise VLLMValidationError( f"Unsupported chat content part type: {part_type!r}. " f"Supported types: {', '.join(supported)}.", parameter="type", value=part_type, ) if wrap_dicts: if modality == "prompt_embeds": # Chat templates don't know about the "prompt_embeds" modality, # emit the single sentinel token as text so the template renders # it inline. The renderer later expands it to N tokens post-tokenize. return {"type": "text", "text": PROMPT_EMBEDS_PLACEHOLDER_TOKEN} result = {"type": modality} result.update(_collect_extra_fields(cast(dict[str, Any], part))) return result if modality == "prompt_embeds": # Emit the renderer token inline regardless of `interleave_strings`, # prompt_embeds are spliced at the token offset so position matters. # Falling back to front-padding via `missing_placeholders` would # reorder them relative to surrounding text. return PROMPT_EMBEDS_PLACEHOLDER_TOKEN return MODALITY_PLACEHOLDERS_MAP[modality] if interleave_strings else None _CLAUDE_CODE_BILLING_HEADER = "x-anthropic-billing-header" def _strip_claude_code_billing_header( parts: Iterable[ChatCompletionContentPartParam], ) -> list[ChatCompletionContentPartParam]: """Drop Claude Code's attribution header line from system text parts. Its ``cch`` value changes on every request, so a prompt that keeps it misses the prefix cache from the header on. Gateways that translate Anthropic requests to chat completions keep it; ``/v1/messages`` drops the same block. """ out: list[ChatCompletionContentPartParam] = [] for part in parts: if isinstance(part, str): text: object = part elif isinstance(part, dict) and part.get("type") == "text": text = part.get("text") else: text = None if not (isinstance(text, str) and text.startswith(_CLAUDE_CODE_BILLING_HEADER)): out.append(part) continue rest = text.partition("\n")[2] if not rest: continue out.append( cast(ChatCompletionContentPartParam, rest) if isinstance(part, str) else cast(ChatCompletionContentPartParam, {**part, "text": rest}) ) return out def parse_chat_message_content( message: ChatCompletionMessageParam, mm_tracker: BaseMultiModalItemTracker, content_format: ChatTemplateContentFormat, interleave_strings: bool, mm_processor_kwargs: dict[str, Any] | None = None, ) -> list[ConversationMessage]: role = message["role"] content = message.get("content") reasoning = message.get("reasoning") if content is None: content = [] elif isinstance(content, str): content = [ChatCompletionContentPartTextParam(type="text", text=content)] if role == "system": content = _strip_claude_code_billing_header(content) # type: ignore[arg-type] result = parse_chat_message_content_parts( role, content, # type: ignore mm_tracker, wrap_dicts=(content_format == "openai"), interleave_strings=interleave_strings, mm_processor_kwargs=mm_processor_kwargs, ) for result_msg in result: if role == "assistant": parsed_msg = _AssistantParser(message) # The 'tool_calls' is not None check ensures compatibility. # It's needed only if downstream code doesn't strictly # follow the OpenAI spec. if "tool_calls" in parsed_msg and parsed_msg["tool_calls"] is not None: result_msg["tool_calls"] = list(parsed_msg["tool_calls"]) # Include reasoning if present for interleaved thinking. if reasoning is not None: result_msg["reasoning"] = cast(str, reasoning) result_msg["reasoning_content"] = cast( str, reasoning ) # keep compatibility elif role == "tool": parsed_msg = _ToolParser(message) if "tool_call_id" in parsed_msg: result_msg["tool_call_id"] = parsed_msg["tool_call_id"] # Normalize tool message content from OpenAI array format to plain # string. Clients like Claude Code / Cursor send tool results as # [{"type": "text", "text": "..."}], but most chat templates only # handle string content for tool messages. # However, tool_reference items must be preserved as structured # dicts for the chat template to expand them. msg_content = result_msg.get("content") if isinstance(msg_content, list): has_non_text = any( isinstance(item, dict) and item.get("type") != "text" for item in msg_content ) if has_non_text: # Keep structured content (e.g., tool_reference) result_msg["content"] = msg_content else: texts = [ item.get("text", "") for item in msg_content if isinstance(item, dict) and item.get("type") == "text" ] result_msg["content"] = "\n".join(texts) if texts else "" if "name" in message and isinstance(message["name"], str): result_msg["name"] = message["name"] if "task" in message and isinstance(message["task"], str): result_msg["task"] = message["task"] if role == "developer": result_msg["tools"] = message.get("tools", None) return result def _postprocess_messages(messages: list[ConversationMessage]) -> None: # per the Transformers docs & maintainers, tool call arguments in # assistant-role messages with tool_calls need to be dicts not JSON str - # this is how tool-use chat templates will expect them moving forwards # so, for messages that have tool_calls, parse the string (which we get # from openAI format) to dict for message in messages: if message["role"] == "assistant" and "tool_calls" in message: tool_calls = message.get("tool_calls") if not isinstance(tool_calls, list): continue if len(tool_calls) == 0: # Drop empty tool_calls to keep templates on the normal assistant path. message.pop("tool_calls", None) continue for item in tool_calls: if not isinstance(item, dict): raise VLLMValidationError( "assistant tool_calls entries must be objects.", parameter="tool_calls", ) function = item.get("function") if item.get("type", "function") != "function" or not isinstance( function, dict ): raise VLLMValidationError( "chat completions only support assistant tool_calls " "of type 'function'.", parameter="tool_calls", ) # if arguments is None or empty string, set to {} if content := function.get("arguments"): if isinstance(content, dict): parsed = content else: if isinstance(content, str): try: parsed = json.loads(content) except json.JSONDecodeError: # A malformed `arguments` string lives in # conversation history, so failing the request # here would fail every subsequent turn too and # leave the conversation unrecoverable. Coerce # to an empty object so the turn can proceed. logger.warning( "Tool call has arguments that are not valid " "JSON (%d chars); coercing to an empty object " "so the conversation can continue.", len(content), ) parsed = None else: parsed = content if not isinstance(parsed, dict): if parsed is not None: # Valid JSON, but not an object (e.g. "[]", # "42", "true"). # Chat templates require a mapping. logger.warning( "Tool call arguments decoded to %s, not a " "JSON object; coercing to an empty object.", type(parsed).__name__, ) parsed = {} function["arguments"] = parsed else: function["arguments"] = {} def parse_chat_messages( messages: list[ChatCompletionMessageParam], model_config: ModelConfig, content_format: ChatTemplateContentFormat, media_io_kwargs: dict[str, dict[str, Any]] | None = None, mm_processor_kwargs: dict[str, Any] | None = None, ) -> tuple[ list[ConversationMessage], MultiModalDataDict | None, MultiModalUUIDDict | None, ]: conversation: list[ConversationMessage] = [] mm_tracker = MultiModalItemTracker( model_config, media_io_kwargs=media_io_kwargs, ) for msg in messages: sub_messages = parse_chat_message_content( msg, mm_tracker, content_format, interleave_strings=( content_format == "string" and model_config.multimodal_config is not None and model_config.multimodal_config.interleave_mm_strings ), mm_processor_kwargs=mm_processor_kwargs, ) conversation.extend(sub_messages) _postprocess_messages(conversation) mm_data, mm_uuids = mm_tracker.resolve_items() return conversation, mm_data, mm_uuids async def parse_chat_messages_async( messages: list[ChatCompletionMessageParam], model_config: ModelConfig, content_format: ChatTemplateContentFormat, media_io_kwargs: dict[str, dict[str, Any]] | None = None, mm_processor_kwargs: dict[str, Any] | None = None, ) -> tuple[ list[ConversationMessage], MultiModalDataDict | None, MultiModalUUIDDict | None, ]: conversation: list[ConversationMessage] = [] mm_tracker = AsyncMultiModalItemTracker( model_config, media_io_kwargs=media_io_kwargs, ) for msg in messages: sub_messages = parse_chat_message_content( msg, mm_tracker, content_format, interleave_strings=( content_format == "string" and model_config.multimodal_config is not None and model_config.multimodal_config.interleave_mm_strings ), mm_processor_kwargs=mm_processor_kwargs, ) conversation.extend(sub_messages) _postprocess_messages(conversation) mm_data, mm_uuids = await mm_tracker.resolve_items() return conversation, mm_data, mm_uuids