[Misc] Reorganize inputs (#35182)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
Cyrus Leung
2026-03-25 10:22:54 -07:00
committed by GitHub
parent 678b3c99e8
commit ba2f0acc2d
142 changed files with 1212 additions and 1342 deletions
+1 -15
View File
@@ -1,16 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from .hasher import MultiModalHasher
from .inputs import (
BatchedTensorInputs,
ModalityData,
MultiModalDataBuiltins,
MultiModalDataDict,
MultiModalKwargsItems,
MultiModalPlaceholderDict,
MultiModalUUIDDict,
NestedTensors,
)
from .inputs import BatchedTensorInputs, MultiModalKwargsItems, NestedTensors
from .registry import MultiModalRegistry
MULTIMODAL_REGISTRY = MultiModalRegistry()
@@ -25,13 +16,8 @@ Info:
__all__ = [
"BatchedTensorInputs",
"ModalityData",
"MultiModalDataBuiltins",
"MultiModalDataDict",
"MultiModalHasher",
"MultiModalKwargsItems",
"MultiModalPlaceholderDict",
"MultiModalUUIDDict",
"NestedTensors",
"MULTIMODAL_REGISTRY",
"MultiModalRegistry",
+2 -162
View File
@@ -15,12 +15,11 @@ from typing import (
TypedDict,
Union,
cast,
final,
)
import numpy as np
from PIL.Image import Image
from typing_extensions import NotRequired, TypeVar
from typing_extensions import TypeVar
from vllm.utils.collection_utils import is_list_of
from vllm.utils.import_utils import LazyLoader
@@ -32,14 +31,9 @@ if TYPE_CHECKING:
import torch
import torch.types
from transformers.feature_extraction_utils import BatchFeature
from vllm.inputs.data import _InputOptions
else:
torch = LazyLoader("torch", globals(), "torch")
_InputOptions = dict
_T = TypeVar("_T")
HfImageItem: TypeAlias = Union["Image", np.ndarray, "torch.Tensor"]
"""
@@ -98,15 +92,6 @@ which are treated as audio embeddings;
these are directly passed to the model without HF processing.
"""
ModalityData: TypeAlias = _T | list[_T | None] | None
"""
Either a single data item, or a list of data items. Can only be None if UUID
is provided.
The number of data items allowed per modality is restricted by
`--limit-mm-per-prompt`.
"""
class VisionChunkImage(TypedDict):
"""Represents an image wrapped as a vision chunk."""
@@ -126,46 +111,10 @@ class VisionChunkVideo(TypedDict):
video_idx: int
VisionChunk = VisionChunkImage | VisionChunkVideo
VisionChunk: TypeAlias = VisionChunkImage | VisionChunkVideo
"""A vision chunk is either an image or a video chunk."""
@final
class MultiModalDataBuiltins(TypedDict, total=False):
"""Type annotations for modality types predefined by vLLM."""
image: ModalityData[ImageItem]
"""The input image(s)."""
video: ModalityData[VideoItem]
"""The input video(s)."""
audio: ModalityData[AudioItem]
"""The input audio(s)."""
vision_chunk: ModalityData[VisionChunk]
"""The input visual atom(s) - unified modality for images and video chunks."""
MultiModalDataDict: TypeAlias = Mapping[str, ModalityData[Any]]
"""
A dictionary containing an entry for each modality type to input.
The built-in modalities are defined by
[`MultiModalDataBuiltins`][vllm.multimodal.inputs.MultiModalDataBuiltins].
"""
MultiModalUUIDDict: TypeAlias = Mapping[str, Sequence[str | None] | str]
"""
A dictionary containing user-provided UUIDs for items in each modality.
If a UUID for an item is not provided, its entry will be `None` and
MultiModalHasher will compute a hash for the item.
The UUID will be used to identify the item for all caching purposes
(input processing caching, embedding caching, prefix caching, etc).
"""
@dataclass(frozen=True)
class PlaceholderRange:
"""
@@ -1048,112 +997,3 @@ MultiModalKwargsOptionalItems: TypeAlias = (
MultiModalKwargsItems[MultiModalKwargsItem]
| MultiModalKwargsItems[MultiModalKwargsItem | None]
)
MultiModalHashes = dict[str, list[str]]
"""
A dictionary containing per-item hashes for each modality.
"""
MultiModalPlaceholderDict: TypeAlias = Mapping[str, Sequence[PlaceholderRange]]
"""
A dictionary containing per-item placeholder ranges for each modality.
"""
class MultiModalInputs(_InputOptions):
"""
Represents the outputs of
[`BaseMultiModalProcessor`][vllm.multimodal.processing.BaseMultiModalProcessor],
ready to be passed to vLLM internals.
"""
type: Literal["multimodal"]
"""The type of inputs."""
prompt_token_ids: list[int]
"""The processed token IDs which includes placeholder tokens."""
prompt: NotRequired[str]
"""The prompt text corresponding to the token IDs, if available."""
mm_kwargs: MultiModalKwargsOptionalItems
"""Keyword arguments to be directly passed to the model after batching."""
mm_hashes: MultiModalHashes
"""The hashes of the multi-modal data."""
mm_placeholders: MultiModalPlaceholderDict
"""
For each modality, information about the placeholder tokens in
`prompt_token_ids`.
"""
def mm_inputs(
prompt_token_ids: list[int],
mm_kwargs: MultiModalKwargsOptionalItems,
mm_hashes: MultiModalHashes,
mm_placeholders: MultiModalPlaceholderDict,
*,
prompt: str | None = None,
cache_salt: str | None = None,
) -> MultiModalInputs:
inputs = MultiModalInputs(
type="multimodal",
prompt_token_ids=prompt_token_ids,
mm_kwargs=mm_kwargs,
mm_hashes=mm_hashes,
mm_placeholders=mm_placeholders,
)
if prompt is not None:
inputs["prompt"] = prompt
if cache_salt is not None:
inputs["cache_salt"] = cache_salt
return inputs
class MultiModalEncDecInputs(MultiModalInputs):
"""
Represents the outputs of
[`EncDecMultiModalProcessor`][vllm.multimodal.processing.EncDecMultiModalProcessor]
ready to be passed to vLLM internals.
Note: Even text-only encoder-decoder models are currently implemented
as multi-modal models for convenience.
(Example: https://github.com/vllm-project/bart-plugin)
"""
encoder_prompt_token_ids: list[int]
"""The processed token IDs of the encoder prompt."""
encoder_prompt: NotRequired[str]
"""The prompt text corresponding to the encoder token IDs, if available."""
def mm_enc_dec_inputs(
encoder_inputs: MultiModalInputs,
decoder_prompt_token_ids: list[int],
*,
decoder_prompt: str | None = None,
) -> MultiModalEncDecInputs:
inputs = MultiModalEncDecInputs(
type="multimodal",
prompt_token_ids=decoder_prompt_token_ids,
encoder_prompt_token_ids=encoder_inputs["prompt_token_ids"],
mm_kwargs=encoder_inputs["mm_kwargs"],
mm_hashes=encoder_inputs["mm_hashes"],
mm_placeholders=encoder_inputs["mm_placeholders"],
)
if decoder_prompt is not None:
inputs["prompt"] = decoder_prompt
if "prompt" in encoder_inputs:
inputs["encoder_prompt"] = encoder_inputs["prompt"]
if "cache_salt" in encoder_inputs:
inputs["cache_salt"] = encoder_inputs["cache_salt"]
return inputs
+6 -8
View File
@@ -19,6 +19,7 @@ import numpy as np
import torch
from typing_extensions import assert_never
from vllm.inputs import ModalityData, MultiModalDataDict, MultiModalUUIDDict
from vllm.utils.collection_utils import is_list_of
from vllm.utils.import_utils import LazyLoader
@@ -29,11 +30,8 @@ from .inputs import (
HfImageItem,
HfVideoItem,
ImageItem,
ModalityData,
MultiModalDataDict,
MultiModalFieldConfig,
MultiModalKwargsItems,
MultiModalUUIDDict,
VideoItem,
)
from .media import MediaWithBytes
@@ -407,8 +405,8 @@ _D = TypeVar("_D", bound=ModalityDataItems[Any, Any])
class MultiModalDataItems(UserDict[str, ModalityDataItems[Any, Any]]):
"""
As [`MultiModalDataDict`][vllm.multimodal.inputs.MultiModalDataDict], but
normalized such that each entry corresponds to a list.
A normalized [`MultiModalDataDict`][vllm.inputs.MultiModalDataDict]
such that each entry corresponds to a list.
"""
def select(self, modalities: Set[str]):
@@ -477,7 +475,7 @@ ModalityDataParser: TypeAlias = Callable[
class MultiModalDataParser:
"""
Parses [`MultiModalDataDict`][vllm.multimodal.inputs.MultiModalDataDict]
Parses [`MultiModalDataDict`][vllm.inputs.MultiModalDataDict]
into [`MultiModalDataItems`][vllm.multimodal.parse.MultiModalDataItems].
Args:
@@ -695,8 +693,8 @@ class MultiModalDataParser:
MultiModalUUIDItems: TypeAlias = dict[str, Sequence[str | None]]
"""
As [`MultiModalUUIDDict`][vllm.multimodal.inputs.MultiModalUUIDDict], but
normalized such that each entry corresponds to a list.
A normalized [`MultiModalUUIDDict`][vllm.inputs.MultiModalUUIDDict]
such that each entry corresponds to a list.
"""
+6 -4
View File
@@ -11,15 +11,14 @@ from typing import TYPE_CHECKING, Any, overload
import torch
from typing_extensions import TypeVar
from vllm.inputs import MultiModalDataDict
from vllm.logger import init_logger
from vllm.multimodal.inputs import MultiModalDataDict
from vllm.multimodal.parse import (
DictEmbeddingItems,
EmbeddingItems,
MultiModalDataItems,
MultiModalDataParser,
)
from vllm.renderers import TokenizeParams
from vllm.tokenizers import TokenizerLike
from vllm.transformers_utils.processor import cached_processor_from_config
from vllm.utils.func_utils import get_allowed_kwarg_only_overrides
@@ -32,12 +31,14 @@ if TYPE_CHECKING:
from transformers.processing_utils import ProcessorMixin
from vllm.config import ModelConfig
from vllm.renderers import TokenizeParams
else:
PretrainedConfig = object
BatchFeature = object
ProcessorMixin = object
ModelConfig = object
TokenizeParams = object
logger = init_logger(__name__)
@@ -339,6 +340,8 @@ class BaseProcessingInfo:
def get_default_tok_params(self) -> TokenizeParams:
"""Construct the default parameters for tokenization."""
from vllm.renderers import TokenizeParams
model_config = self.ctx.model_config
encoder_config = model_config.encoder_config or {}
@@ -451,8 +454,7 @@ class BaseProcessingInfo:
validate: bool = True,
) -> MultiModalDataItems:
"""
Normalize
[`MultiModalDataDict`][vllm.multimodal.inputs.MultiModalDataDict]
Normalize [`MultiModalDataDict`][vllm.inputs.MultiModalDataDict]
to [`MultiModalDataItems`][vllm.multimodal.parse.MultiModalDataItems]
before passing them to
[`_get_hf_mm_data`][vllm.multimodal.processing.BaseMultiModalProcessor._get_hf_mm_data].
+1 -1
View File
@@ -14,9 +14,9 @@ from vllm.config.multimodal import (
ImageDummyOptions,
VideoDummyOptions,
)
from vllm.inputs import MultiModalDataDict
from vllm.logger import init_logger
from ..inputs import MultiModalDataDict
from .context import BaseProcessingInfo
from .inputs import ProcessorInputs
+3 -2
View File
@@ -3,8 +3,9 @@
from collections.abc import Mapping
from dataclasses import dataclass, field
from vllm.inputs import MultiModalHashes
from ..hasher import MultiModalHasher
from ..inputs import MultiModalHashes
from ..parse import MultiModalDataItems, MultiModalUUIDItems
@@ -26,7 +27,7 @@ class ProcessorInputs:
mm_uuid_items = self.mm_uuid_items or {}
hf_processor_mm_kwargs = self.hf_processor_mm_kwargs
mm_hashes: MultiModalHashes = {}
mm_hashes = dict[str, list[str]]()
hasher = MultiModalHasher
for modality, data_items in mm_data_items.items():
+14 -19
View File
@@ -6,34 +6,29 @@ from collections.abc import Callable, Generator, ItemsView, Iterable, Mapping, S
from dataclasses import dataclass, field, replace
from enum import Enum
from functools import lru_cache
from typing import (
TYPE_CHECKING,
Generic,
NamedTuple,
Protocol,
TypeAlias,
cast,
)
from typing import TYPE_CHECKING, Generic, NamedTuple, Protocol, TypeAlias, cast
import regex as re
import torch
from typing_extensions import TypeVar, assert_never
from vllm.inputs import (
MultiModalEncDecInput,
MultiModalHashes,
MultiModalInput,
mm_enc_dec_input,
mm_input,
)
from vllm.logger import init_logger
from vllm.tokenizers import TokenizerLike
from vllm.utils.collection_utils import flatten_2d_lists, full_groupby
from ..inputs import (
MultiModalEncDecInputs,
MultiModalFieldConfig,
MultiModalHashes,
MultiModalInputs,
MultiModalKwargsItem,
MultiModalKwargsItems,
MultiModalKwargsOptionalItems,
PlaceholderRange,
mm_enc_dec_inputs,
mm_inputs,
)
from ..parse import (
DictEmbeddingItems,
@@ -994,7 +989,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
mm_items: MultiModalDataItems,
mm_uuid_items: MultiModalUUIDItems | None = None,
hf_processor_mm_kwargs: Mapping[str, object] | None = None,
) -> MultiModalInputs:
) -> MultiModalInput:
processor_inputs = ProcessorInputs(
prompt,
mm_items,
@@ -1638,7 +1633,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
self,
inputs: ProcessorInputs,
timing_ctx: TimingContext,
) -> MultiModalInputs:
) -> MultiModalInput:
"""
Process multi-modal inputs to be used in vLLM.
@@ -1673,7 +1668,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
for modality, placeholders in mm_placeholders.items()
}
return mm_inputs(
return mm_input(
prompt_token_ids=prompt_ids,
mm_kwargs=mm_info.kwargs,
mm_hashes=mm_info.hashes,
@@ -1708,7 +1703,7 @@ class EncDecMultiModalProcessor(BaseMultiModalProcessor[_I]):
self,
prompt: str | list[int],
mm_items: MultiModalDataItems,
encoder_inputs: MultiModalInputs,
encoder_inputs: MultiModalInput,
):
tokenizer = self.info.get_tokenizer()
decoder_prompt_raw = self.create_decoder_prompt(prompt, mm_items)
@@ -1721,7 +1716,7 @@ class EncDecMultiModalProcessor(BaseMultiModalProcessor[_I]):
decoder_prompt_text = None
decoder_prompt_ids = decoder_prompt_raw
return mm_enc_dec_inputs(
return mm_enc_dec_input(
encoder_inputs,
decoder_prompt_ids,
decoder_prompt=decoder_prompt_text,
@@ -1731,7 +1726,7 @@ class EncDecMultiModalProcessor(BaseMultiModalProcessor[_I]):
self,
inputs: ProcessorInputs,
timing_ctx: TimingContext,
) -> MultiModalEncDecInputs:
) -> MultiModalEncDecInput:
"""
Process multi-modal inputs to be used in vLLM.
The main processing steps are modified to fit encoder-decoder model:
+2 -2
View File
@@ -7,6 +7,7 @@ from dataclasses import dataclass
from multiprocessing.synchronize import Lock as LockType
from typing import TYPE_CHECKING, Generic, Literal, Protocol, TypeVar, cast
from vllm.inputs import MultiModalInput
from vllm.logger import init_logger
from vllm.tokenizers import TokenizerLike, cached_tokenizer_from_config
@@ -19,7 +20,6 @@ from .cache import (
ShmObjectStoreReceiverCache,
ShmObjectStoreSenderCache,
)
from .inputs import MultiModalInputs
from .processing import (
BaseDummyInputsBuilder,
BaseMultiModalProcessor,
@@ -220,7 +220,7 @@ class MultiModalRegistry:
*,
cache: BaseMultiModalProcessorCache | None = None,
processor: BaseMultiModalProcessor | None = None,
) -> MultiModalInputs:
) -> MultiModalInput:
"""
Create dummy data for profiling the memory usage of a model.
+3 -3
View File
@@ -12,6 +12,7 @@ import numpy.typing as npt
from PIL import Image
from typing_extensions import deprecated
from vllm.inputs import MultiModalPlaceholders
from vllm.utils.import_utils import LazyLoader
from .hasher import MultiModalHasher
@@ -19,7 +20,6 @@ from .inputs import (
BatchedTensorInputs,
MultiModalFieldElem,
MultiModalKwargsItem,
MultiModalPlaceholderDict,
MultiModalSharedField,
)
from .media import AudioMediaIO, ImageMediaIO, MediaConnector, VideoMediaIO
@@ -110,10 +110,10 @@ def encode_video_url(
def argsort_mm_positions(
mm_positions: MultiModalPlaceholderDict,
mm_positions: MultiModalPlaceholders,
) -> list[tuple[str, int]]:
"""
Given a `MultiModalPlaceholderDict`, output a sequence of keys to
Given a `MultiModalPlaceholders`, output a sequence of keys to
sort the dictionary by `offset` (starting index in the input sequence)
in ascending order.