# SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import asyncio from abc import ABC, abstractmethod from collections import defaultdict from collections.abc import Awaitable, Callable from functools import cached_property from itertools import accumulate from typing import ( TYPE_CHECKING, Any, Final, Generic, TypeAlias, TypeVar, cast, ) from vllm.config import ModelConfig from vllm.exceptions import VLLMValidationError from vllm.inputs import MultiModalDataDict, MultiModalUUIDDict from vllm.logger import init_logger from vllm.model_executor.models import SupportsMultiModal from vllm.multimodal import MULTIMODAL_REGISTRY from vllm.multimodal.inputs import ( MultiModalBatchedField, MultiModalFlatField, MultiModalSharedField, VisionChunk, VisionChunkImage, VisionChunkVideo, ) from vllm.multimodal.media.base import MediaDecodedBudget from vllm.multimodal.processing import BaseMultiModalProcessor from vllm.transformers_utils.processor import get_video_processor_cls_name from vllm.utils import random_uuid from vllm.utils.collection_utils import is_list_of, is_list_of_numbers from vllm.utils.import_utils import LazyLoader from ...connections import MediaRequestDownloadBudget from .content_parser import ( AsyncMultiModalContentParser, BaseMultiModalContentParser, MultiModalContentParser, ) from .types import ModalityStr _T = TypeVar("_T") _AsyncMultiModalItem: TypeAlias = Callable[[], Awaitable[tuple[object, str | None]]] if TYPE_CHECKING: import torch import transformers else: transformers = LazyLoader("transformers", globals(), "transformers") torch = LazyLoader("torch", globals(), "torch") logger = init_logger(__name__) _REQUIRE_MM_PROCESSOR_ERROR: Final[str] = ( "Resolving modality {modality!r} requires a multimodal processor " "but none is available." ) # Backward compatibility for single item input class _BatchedSingleItemField(MultiModalSharedField): pass def _detect_field( tensors: list[torch.Tensor], mm_processor: BaseMultiModalProcessor, ): first_item = tensors[0] hidden_size = mm_processor.info.ctx.model_config.get_inputs_embeds_size() if ( len(tensors) == 1 and first_item.ndim == 3 and first_item.shape[0] == 1 and first_item.shape[-1] == hidden_size ): logger.warning( "Batched multi-modal embedding inputs are deprecated for Chat API. " "Please pass a separate content part for each multi-modal item." ) return _BatchedSingleItemField(batch_size=1) first_shape = first_item.shape if all(t.shape == first_shape for t in tensors): return MultiModalBatchedField() size_per_item = [len(tensor) for tensor in tensors] slice_idxs = [0, *accumulate(size_per_item)] slices = [ (slice(slice_idxs[i], slice_idxs[i + 1]),) for i in range(len(size_per_item)) ] return MultiModalFlatField(slices=slices) def _merge_embeds( data_items: list[dict[str, "torch.Tensor"]], mm_processor: BaseMultiModalProcessor, ): if not data_items: return {} first_keys = set(data_items[0].keys()) if any(set(item.keys()) != first_keys for item in data_items[1:]): raise VLLMValidationError( "All dictionaries in the list of embeddings must have the same keys." ) fields = { key: _detect_field([item[key] for item in data_items], mm_processor) for key in first_keys } data_merged = { key: field._reduce_data([item[key] for item in data_items], pin_memory=False) for key, field in fields.items() } try: # TODO: Support per-request mm_processor_kwargs parsed_configs = mm_processor._get_mm_fields_config( transformers.BatchFeature(data_merged), {}, ) parsed_fields = {key: parsed_configs[key].field for key in first_keys} keys_to_update = [ key for key in first_keys if ( fields[key] != parsed_fields[key] and not isinstance(fields[key], _BatchedSingleItemField) ) ] for key in keys_to_update: data_merged[key] = parsed_fields[key]._reduce_data( [item[key] for item in data_items], pin_memory=False ) except Exception: logger.exception( "Error when parsing merged embeddings. " "Falling back to auto-detected fields." ) return data_merged def _parse_metadata_array(key: str, value: list, metadata_fields: set[str]): if key not in metadata_fields: raise VLLMValidationError(f"JSON arrays are only supported for metadata: {key}") if not is_list_of_numbers(value): raise VLLMValidationError(f"Metadata {key} must be a finite numeric array.") dtype = torch.float64 if any(isinstance(v, float) for v in value) else torch.int64 try: return torch.tensor(value, dtype=dtype) except (ValueError, TypeError, OverflowError, RuntimeError) as error: raise VLLMValidationError(f"Invalid metadata array: {key}") from error def _get_embeds_data( modality: str, data_items: list[Any], mm_processor: BaseMultiModalProcessor, ): if len(data_items) == 0: return data_items if all(item is None for item in data_items): return data_items if is_list_of(data_items, torch.Tensor): embeds_key = f"{modality}_embeds" dict_items = [{embeds_key: item} for item in data_items] return _merge_embeds(dict_items, mm_processor)[embeds_key] if is_list_of(data_items, dict): metadata_fields = mm_processor.info.data_parser.placeholder_metadata_fields( modality ) data_items = [ { key: _parse_metadata_array(key, value, metadata_fields) if isinstance(value, list) else value for key, value in item.items() } for item in data_items ] return _merge_embeds(data_items, mm_processor) raise NotImplementedError(type(data_items)) class BaseMultiModalItemTracker(ABC, Generic[_T]): """Tracks multi-modal items in a given request and ensures that the number of multi-modal items in a given request does not exceed the configured maximum per prompt. """ def __init__( self, model_config: ModelConfig, media_io_kwargs: dict[str, dict[str, Any]] | None = None, ): super().__init__() self._model_config = model_config self._media_io_kwargs = media_io_kwargs self._items_by_modality = defaultdict[str, list[_T]](list) # Track original modality for each vision_chunk item (image or video) self._modality_order = defaultdict[str, list[str]](list) self.decoded_budget = MediaDecodedBudget() self.download_budget = MediaRequestDownloadBudget() @cached_property def use_unified_vision_chunk_modality(self) -> bool: """Check if model uses unified vision_chunk modality for images/videos.""" return getattr(self._model_config.hf_config, "use_unified_vision_chunk", False) @property def model_config(self) -> ModelConfig: return self._model_config @cached_property def model_cls(self) -> type[SupportsMultiModal]: from vllm.model_executor.model_loader import get_model_cls model_cls = get_model_cls(self.model_config) return cast(type[SupportsMultiModal], model_cls) @property def media_io_kwargs(self) -> dict[str, dict[str, Any]] | None: return self._media_io_kwargs or ( self._model_config.multimodal_config.media_io_kwargs if self._model_config.multimodal_config else None ) @property def allowed_local_media_path(self): return self._model_config.allowed_local_media_path @property def allowed_media_domains(self): return self._model_config.allowed_media_domains @property def mm_registry(self): return MULTIMODAL_REGISTRY @cached_property def mm_processor(self): return self.mm_registry.create_processor(self.model_config) @property def video_processor_name(self) -> str | None: return get_video_processor_cls_name(self.model_config) def add(self, modality: ModalityStr, item: _T) -> str | None: """Add a multi-modal item to the current prompt and returns the placeholder string to use, if any. An optional uuid can be added which serves as a unique identifier of the media. Note: `prompt_embeds` bypass MM-processor validation because they are pre-computed embeddings that do not go through any HF processor, encoder, or model-specific placeholder logic. The corresponding placeholder string is managed by the parser via `_add_placeholder`, so we return None here. """ add_info = self._validate_add(modality) if add_info is None: self._items_by_modality["prompt_embeds"].append(item) return None input_modality, original_modality, use_vision_chunk, num_items = add_info # Track original modality for vision_chunk items if use_vision_chunk: self._items_by_modality[input_modality].append(item) # type: ignore self._modality_order["vision_chunk"].append(original_modality) else: self._items_by_modality[original_modality].append(item) return self.model_cls.get_placeholder_str(modality, num_items) def _validate_add(self, modality: ModalityStr) -> tuple[str, str, bool, int] | None: """Validate that one more item of the modality can be tracked.""" if modality == "prompt_embeds": return None input_modality = modality.replace("_embeds", "") original_modality = modality use_vision_chunk = ( self.use_unified_vision_chunk_modality and original_modality in ["video", "image"] ) # If use_unified_vision_chunk_modality is enabled, # map image/video to vision_chunk if use_vision_chunk: # To avoid validation fail # because models with use_unified_vision_chunk_modality=True # will only accept vision_chunk modality. input_modality = "vision_chunk" num_items = len(self._items_by_modality[input_modality]) + 1 else: num_items = len(self._items_by_modality[original_modality]) + 1 mm_config = self.model_config.multimodal_config if ( mm_config is not None and mm_config.enable_mm_embeds and mm_config.get_limit_per_prompt(input_modality) == 0 and original_modality.endswith("_embeds") ): # Skip validation: embeddings bypass limit when enable_mm_embeds=True pass else: self.mm_processor.info.validate_num_items(input_modality, num_items) return input_modality, original_modality, use_vision_chunk, num_items @abstractmethod def create_parser( self, mm_processor_kwargs: dict[str, Any] | None = None ) -> "BaseMultiModalContentParser": raise NotImplementedError def _resolve_vision_chunk_items( vision_chunk_items: list[tuple[object, str | None]], mm_processor: BaseMultiModalProcessor, vision_chunks_modality_order: list[str], ): # Process vision_chunk items - extract from (data, modality) tuples # and convert to VisionChunk types with proper UUID handling vision_chunks_uuids = [uuid for data, uuid in vision_chunk_items] assert len(vision_chunk_items) == len(vision_chunks_modality_order), ( f"vision_chunk items ({len(vision_chunk_items)}) and " f"modality_order ({len(vision_chunks_modality_order)}) must have same length" ) processed_chunks: list[VisionChunk] = [] video_idx = 0 for inner_modality, (data, uuid) in zip( vision_chunks_modality_order, vision_chunk_items ): if inner_modality == "image": # Cast data to proper type for image # Use .media (PIL.Image) directly to avoid redundant # bytes→PIL conversion in media_processor if hasattr(data, "media"): image_data = data.media # type: ignore[union-attr] processed_chunks.append( VisionChunkImage(type="image", image=image_data, uuid=uuid) ) else: processed_chunks.append(data) # type: ignore[arg-type] elif inner_modality == "video": # For video, we may need to split into chunks # if processor supports it # For now, just wrap as a video chunk placeholder if hasattr(mm_processor, "split_video_chunks") and data is not None: try: video_uuid = uuid or random_uuid() # video await result is (video_data, video_meta) tuple if isinstance(data, tuple) and len(data) >= 1: video_data = data[0] else: video_data = data video_chunks = mm_processor.split_video_chunks(video_data) for i, vc in enumerate(video_chunks): processed_chunks.append( VisionChunkVideo( type="video_chunk", video_chunk=vc["video_chunk"], uuid=f"{video_uuid}-{i}", video_idx=video_idx, prompt=vc["prompt"], ) ) video_idx += 1 except Exception as e: logger.warning("Failed to split video chunks: %s", e) processed_chunks.append(data) # type: ignore[arg-type] else: processed_chunks.append(data) # type: ignore[arg-type] return processed_chunks, vision_chunks_uuids def _resolve_items( items_by_modality: dict[str, list[tuple[object, str | None]]], mm_processor: BaseMultiModalProcessor | None, modality_order: dict[str, list[str]], ) -> tuple[MultiModalDataDict, MultiModalUUIDDict]: """Materialize the tracker's per-modality items into `mm_data` / `mm_uuids`. Note: `mm_processor` is `None` for text-only models (no registered HF processor) whose only modality is `prompt_embeds`. Every other modality requires a processor, enforced by the guard below. """ if "image" in items_by_modality and "image_embeds" in items_by_modality: raise VLLMValidationError( "Mixing raw image and embedding inputs is not allowed", parameter="image_embeds", ) if "audio" in items_by_modality and "audio_embeds" in items_by_modality: raise VLLMValidationError( "Mixing raw audio and embedding inputs is not allowed", parameter="audio_embeds", ) if "video" in items_by_modality and "video_embeds" in items_by_modality: raise VLLMValidationError( "Mixing raw video and embedding inputs is not allowed", parameter="video_embeds", ) # `prompt_embeds` bypasses HF MM processors. Every other modality requires one. processor_modalities = items_by_modality.keys() - {"prompt_embeds"} if processor_modalities and mm_processor is None: raise RuntimeError( _REQUIRE_MM_PROCESSOR_ERROR.format(modality=processor_modalities) ) mm_data = {} mm_uuids = {} if "image_embeds" in items_by_modality: assert mm_processor is not None mm_data["image"] = _get_embeds_data( "image", [data for data, uuid in items_by_modality["image_embeds"]], mm_processor, ) mm_uuids["image"] = [uuid for data, uuid in items_by_modality["image_embeds"]] if "image" in items_by_modality: mm_data["image"] = [data for data, uuid in items_by_modality["image"]] mm_uuids["image"] = [uuid for data, uuid in items_by_modality["image"]] if "audio_embeds" in items_by_modality: assert mm_processor is not None mm_data["audio"] = _get_embeds_data( "audio", [data for data, uuid in items_by_modality["audio_embeds"]], mm_processor, ) mm_uuids["audio"] = [uuid for data, uuid in items_by_modality["audio_embeds"]] if "audio" in items_by_modality: mm_data["audio"] = [data for data, uuid in items_by_modality["audio"]] mm_uuids["audio"] = [uuid for data, uuid in items_by_modality["audio"]] if "video" in items_by_modality: mm_data["video"] = [data for data, uuid in items_by_modality["video"]] mm_uuids["video"] = [uuid for data, uuid in items_by_modality["video"]] if "video_embeds" in items_by_modality: assert mm_processor is not None mm_data["video"] = _get_embeds_data( "video", [data for data, uuid in items_by_modality["video_embeds"]], mm_processor, ) mm_uuids["video"] = [uuid for data, uuid in items_by_modality["video_embeds"]] if "vision_chunk" in items_by_modality: assert mm_processor is not None # Process vision_chunk items - extract from (data, modality) tuples # and convert to VisionChunk types with proper UUID handling processed_chunks, vision_chunk_uuids = _resolve_vision_chunk_items( items_by_modality["vision_chunk"], mm_processor, modality_order.get("vision_chunk", []), ) mm_data["vision_chunk"] = processed_chunks mm_uuids["vision_chunk"] = vision_chunk_uuids if "prompt_embeds" in items_by_modality: mm_data["prompt_embeds"] = [ data for data, _uuid in items_by_modality["prompt_embeds"] ] return mm_data, mm_uuids class MultiModalItemTracker(BaseMultiModalItemTracker[tuple[object, str | None]]): def resolve_items( self, ) -> tuple[MultiModalDataDict | None, MultiModalUUIDDict | None]: if not self._items_by_modality: return None, None # Text-only models (`is_multimodal_model=False`) with inputs of # modality `prompt_embeds` have no MM processor since `prompt_embeds` are # pre-computed and require no processing, so we pass `None`. mm_processor = ( self.mm_processor if self._model_config.is_multimodal_model else None ) return _resolve_items( dict(self._items_by_modality), mm_processor, self._modality_order, ) def create_parser( self, mm_processor_kwargs: dict[str, Any] | None = None ) -> "BaseMultiModalContentParser": return MultiModalContentParser(self, mm_processor_kwargs=mm_processor_kwargs) class AsyncMultiModalItemTracker(BaseMultiModalItemTracker[_AsyncMultiModalItem]): async def resolve_items( self, ) -> tuple[MultiModalDataDict | None, MultiModalUUIDDict | None]: if not self._items_by_modality: return None, None # Fetch all modalities together. Each tracked item is already an # independent awaitable, and the async connector offloads blocking # decode work, so waiting for one modality before starting the next # needlessly adds their latency. # Keep the original group and item order when rebuilding the result. item_groups = list(self._items_by_modality.items()) items = [item for _, group in item_groups for item in group] results = await asyncio.gather( *(item() for item in items), return_exceptions=True ) for result in results: if isinstance(result, BaseException): # Gathering with return_exceptions=True lets every task finish # (or itself fail) before we raise, instead of abandoning # still-in-flight fetches (real network/thread-pool work) the # moment the first one fails. raise result resolved_items_by_modality: dict[str, list[Any]] = {} result_idx = 0 for modality, group in item_groups: next_result_idx = result_idx + len(group) resolved_items_by_modality[modality] = results[result_idx:next_result_idx] result_idx = next_result_idx mm_processor = ( self.mm_processor if self._model_config.is_multimodal_model else None ) return _resolve_items( resolved_items_by_modality, mm_processor, self._modality_order, ) def create_parser( self, mm_processor_kwargs: dict[str, Any] | None = None ) -> "BaseMultiModalContentParser": return AsyncMultiModalContentParser( self, mm_processor_kwargs=mm_processor_kwargs )