[Misc] Reorganize inputs (#35182)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
@@ -1,16 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from .hasher import MultiModalHasher
|
||||
from .inputs import (
|
||||
BatchedTensorInputs,
|
||||
ModalityData,
|
||||
MultiModalDataBuiltins,
|
||||
MultiModalDataDict,
|
||||
MultiModalKwargsItems,
|
||||
MultiModalPlaceholderDict,
|
||||
MultiModalUUIDDict,
|
||||
NestedTensors,
|
||||
)
|
||||
from .inputs import BatchedTensorInputs, MultiModalKwargsItems, NestedTensors
|
||||
from .registry import MultiModalRegistry
|
||||
|
||||
MULTIMODAL_REGISTRY = MultiModalRegistry()
|
||||
@@ -25,13 +16,8 @@ Info:
|
||||
|
||||
__all__ = [
|
||||
"BatchedTensorInputs",
|
||||
"ModalityData",
|
||||
"MultiModalDataBuiltins",
|
||||
"MultiModalDataDict",
|
||||
"MultiModalHasher",
|
||||
"MultiModalKwargsItems",
|
||||
"MultiModalPlaceholderDict",
|
||||
"MultiModalUUIDDict",
|
||||
"NestedTensors",
|
||||
"MULTIMODAL_REGISTRY",
|
||||
"MultiModalRegistry",
|
||||
|
||||
+2
-162
@@ -15,12 +15,11 @@ from typing import (
|
||||
TypedDict,
|
||||
Union,
|
||||
cast,
|
||||
final,
|
||||
)
|
||||
|
||||
import numpy as np
|
||||
from PIL.Image import Image
|
||||
from typing_extensions import NotRequired, TypeVar
|
||||
from typing_extensions import TypeVar
|
||||
|
||||
from vllm.utils.collection_utils import is_list_of
|
||||
from vllm.utils.import_utils import LazyLoader
|
||||
@@ -32,14 +31,9 @@ if TYPE_CHECKING:
|
||||
import torch
|
||||
import torch.types
|
||||
from transformers.feature_extraction_utils import BatchFeature
|
||||
|
||||
from vllm.inputs.data import _InputOptions
|
||||
else:
|
||||
torch = LazyLoader("torch", globals(), "torch")
|
||||
|
||||
_InputOptions = dict
|
||||
|
||||
_T = TypeVar("_T")
|
||||
|
||||
HfImageItem: TypeAlias = Union["Image", np.ndarray, "torch.Tensor"]
|
||||
"""
|
||||
@@ -98,15 +92,6 @@ which are treated as audio embeddings;
|
||||
these are directly passed to the model without HF processing.
|
||||
"""
|
||||
|
||||
ModalityData: TypeAlias = _T | list[_T | None] | None
|
||||
"""
|
||||
Either a single data item, or a list of data items. Can only be None if UUID
|
||||
is provided.
|
||||
|
||||
The number of data items allowed per modality is restricted by
|
||||
`--limit-mm-per-prompt`.
|
||||
"""
|
||||
|
||||
|
||||
class VisionChunkImage(TypedDict):
|
||||
"""Represents an image wrapped as a vision chunk."""
|
||||
@@ -126,46 +111,10 @@ class VisionChunkVideo(TypedDict):
|
||||
video_idx: int
|
||||
|
||||
|
||||
VisionChunk = VisionChunkImage | VisionChunkVideo
|
||||
VisionChunk: TypeAlias = VisionChunkImage | VisionChunkVideo
|
||||
"""A vision chunk is either an image or a video chunk."""
|
||||
|
||||
|
||||
@final
|
||||
class MultiModalDataBuiltins(TypedDict, total=False):
|
||||
"""Type annotations for modality types predefined by vLLM."""
|
||||
|
||||
image: ModalityData[ImageItem]
|
||||
"""The input image(s)."""
|
||||
|
||||
video: ModalityData[VideoItem]
|
||||
"""The input video(s)."""
|
||||
|
||||
audio: ModalityData[AudioItem]
|
||||
"""The input audio(s)."""
|
||||
|
||||
vision_chunk: ModalityData[VisionChunk]
|
||||
"""The input visual atom(s) - unified modality for images and video chunks."""
|
||||
|
||||
|
||||
MultiModalDataDict: TypeAlias = Mapping[str, ModalityData[Any]]
|
||||
"""
|
||||
A dictionary containing an entry for each modality type to input.
|
||||
|
||||
The built-in modalities are defined by
|
||||
[`MultiModalDataBuiltins`][vllm.multimodal.inputs.MultiModalDataBuiltins].
|
||||
"""
|
||||
|
||||
MultiModalUUIDDict: TypeAlias = Mapping[str, Sequence[str | None] | str]
|
||||
"""
|
||||
A dictionary containing user-provided UUIDs for items in each modality.
|
||||
If a UUID for an item is not provided, its entry will be `None` and
|
||||
MultiModalHasher will compute a hash for the item.
|
||||
|
||||
The UUID will be used to identify the item for all caching purposes
|
||||
(input processing caching, embedding caching, prefix caching, etc).
|
||||
"""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class PlaceholderRange:
|
||||
"""
|
||||
@@ -1048,112 +997,3 @@ MultiModalKwargsOptionalItems: TypeAlias = (
|
||||
MultiModalKwargsItems[MultiModalKwargsItem]
|
||||
| MultiModalKwargsItems[MultiModalKwargsItem | None]
|
||||
)
|
||||
|
||||
|
||||
MultiModalHashes = dict[str, list[str]]
|
||||
"""
|
||||
A dictionary containing per-item hashes for each modality.
|
||||
"""
|
||||
|
||||
|
||||
MultiModalPlaceholderDict: TypeAlias = Mapping[str, Sequence[PlaceholderRange]]
|
||||
"""
|
||||
A dictionary containing per-item placeholder ranges for each modality.
|
||||
"""
|
||||
|
||||
|
||||
class MultiModalInputs(_InputOptions):
|
||||
"""
|
||||
Represents the outputs of
|
||||
[`BaseMultiModalProcessor`][vllm.multimodal.processing.BaseMultiModalProcessor],
|
||||
ready to be passed to vLLM internals.
|
||||
"""
|
||||
|
||||
type: Literal["multimodal"]
|
||||
"""The type of inputs."""
|
||||
|
||||
prompt_token_ids: list[int]
|
||||
"""The processed token IDs which includes placeholder tokens."""
|
||||
|
||||
prompt: NotRequired[str]
|
||||
"""The prompt text corresponding to the token IDs, if available."""
|
||||
|
||||
mm_kwargs: MultiModalKwargsOptionalItems
|
||||
"""Keyword arguments to be directly passed to the model after batching."""
|
||||
|
||||
mm_hashes: MultiModalHashes
|
||||
"""The hashes of the multi-modal data."""
|
||||
|
||||
mm_placeholders: MultiModalPlaceholderDict
|
||||
"""
|
||||
For each modality, information about the placeholder tokens in
|
||||
`prompt_token_ids`.
|
||||
"""
|
||||
|
||||
|
||||
def mm_inputs(
|
||||
prompt_token_ids: list[int],
|
||||
mm_kwargs: MultiModalKwargsOptionalItems,
|
||||
mm_hashes: MultiModalHashes,
|
||||
mm_placeholders: MultiModalPlaceholderDict,
|
||||
*,
|
||||
prompt: str | None = None,
|
||||
cache_salt: str | None = None,
|
||||
) -> MultiModalInputs:
|
||||
inputs = MultiModalInputs(
|
||||
type="multimodal",
|
||||
prompt_token_ids=prompt_token_ids,
|
||||
mm_kwargs=mm_kwargs,
|
||||
mm_hashes=mm_hashes,
|
||||
mm_placeholders=mm_placeholders,
|
||||
)
|
||||
|
||||
if prompt is not None:
|
||||
inputs["prompt"] = prompt
|
||||
if cache_salt is not None:
|
||||
inputs["cache_salt"] = cache_salt
|
||||
|
||||
return inputs
|
||||
|
||||
|
||||
class MultiModalEncDecInputs(MultiModalInputs):
|
||||
"""
|
||||
Represents the outputs of
|
||||
[`EncDecMultiModalProcessor`][vllm.multimodal.processing.EncDecMultiModalProcessor]
|
||||
ready to be passed to vLLM internals.
|
||||
|
||||
Note: Even text-only encoder-decoder models are currently implemented
|
||||
as multi-modal models for convenience.
|
||||
(Example: https://github.com/vllm-project/bart-plugin)
|
||||
"""
|
||||
|
||||
encoder_prompt_token_ids: list[int]
|
||||
"""The processed token IDs of the encoder prompt."""
|
||||
|
||||
encoder_prompt: NotRequired[str]
|
||||
"""The prompt text corresponding to the encoder token IDs, if available."""
|
||||
|
||||
|
||||
def mm_enc_dec_inputs(
|
||||
encoder_inputs: MultiModalInputs,
|
||||
decoder_prompt_token_ids: list[int],
|
||||
*,
|
||||
decoder_prompt: str | None = None,
|
||||
) -> MultiModalEncDecInputs:
|
||||
inputs = MultiModalEncDecInputs(
|
||||
type="multimodal",
|
||||
prompt_token_ids=decoder_prompt_token_ids,
|
||||
encoder_prompt_token_ids=encoder_inputs["prompt_token_ids"],
|
||||
mm_kwargs=encoder_inputs["mm_kwargs"],
|
||||
mm_hashes=encoder_inputs["mm_hashes"],
|
||||
mm_placeholders=encoder_inputs["mm_placeholders"],
|
||||
)
|
||||
|
||||
if decoder_prompt is not None:
|
||||
inputs["prompt"] = decoder_prompt
|
||||
if "prompt" in encoder_inputs:
|
||||
inputs["encoder_prompt"] = encoder_inputs["prompt"]
|
||||
if "cache_salt" in encoder_inputs:
|
||||
inputs["cache_salt"] = encoder_inputs["cache_salt"]
|
||||
|
||||
return inputs
|
||||
|
||||
@@ -19,6 +19,7 @@ import numpy as np
|
||||
import torch
|
||||
from typing_extensions import assert_never
|
||||
|
||||
from vllm.inputs import ModalityData, MultiModalDataDict, MultiModalUUIDDict
|
||||
from vllm.utils.collection_utils import is_list_of
|
||||
from vllm.utils.import_utils import LazyLoader
|
||||
|
||||
@@ -29,11 +30,8 @@ from .inputs import (
|
||||
HfImageItem,
|
||||
HfVideoItem,
|
||||
ImageItem,
|
||||
ModalityData,
|
||||
MultiModalDataDict,
|
||||
MultiModalFieldConfig,
|
||||
MultiModalKwargsItems,
|
||||
MultiModalUUIDDict,
|
||||
VideoItem,
|
||||
)
|
||||
from .media import MediaWithBytes
|
||||
@@ -407,8 +405,8 @@ _D = TypeVar("_D", bound=ModalityDataItems[Any, Any])
|
||||
|
||||
class MultiModalDataItems(UserDict[str, ModalityDataItems[Any, Any]]):
|
||||
"""
|
||||
As [`MultiModalDataDict`][vllm.multimodal.inputs.MultiModalDataDict], but
|
||||
normalized such that each entry corresponds to a list.
|
||||
A normalized [`MultiModalDataDict`][vllm.inputs.MultiModalDataDict]
|
||||
such that each entry corresponds to a list.
|
||||
"""
|
||||
|
||||
def select(self, modalities: Set[str]):
|
||||
@@ -477,7 +475,7 @@ ModalityDataParser: TypeAlias = Callable[
|
||||
|
||||
class MultiModalDataParser:
|
||||
"""
|
||||
Parses [`MultiModalDataDict`][vllm.multimodal.inputs.MultiModalDataDict]
|
||||
Parses [`MultiModalDataDict`][vllm.inputs.MultiModalDataDict]
|
||||
into [`MultiModalDataItems`][vllm.multimodal.parse.MultiModalDataItems].
|
||||
|
||||
Args:
|
||||
@@ -695,8 +693,8 @@ class MultiModalDataParser:
|
||||
|
||||
MultiModalUUIDItems: TypeAlias = dict[str, Sequence[str | None]]
|
||||
"""
|
||||
As [`MultiModalUUIDDict`][vllm.multimodal.inputs.MultiModalUUIDDict], but
|
||||
normalized such that each entry corresponds to a list.
|
||||
A normalized [`MultiModalUUIDDict`][vllm.inputs.MultiModalUUIDDict]
|
||||
such that each entry corresponds to a list.
|
||||
"""
|
||||
|
||||
|
||||
|
||||
@@ -11,15 +11,14 @@ from typing import TYPE_CHECKING, Any, overload
|
||||
import torch
|
||||
from typing_extensions import TypeVar
|
||||
|
||||
from vllm.inputs import MultiModalDataDict
|
||||
from vllm.logger import init_logger
|
||||
from vllm.multimodal.inputs import MultiModalDataDict
|
||||
from vllm.multimodal.parse import (
|
||||
DictEmbeddingItems,
|
||||
EmbeddingItems,
|
||||
MultiModalDataItems,
|
||||
MultiModalDataParser,
|
||||
)
|
||||
from vllm.renderers import TokenizeParams
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
from vllm.transformers_utils.processor import cached_processor_from_config
|
||||
from vllm.utils.func_utils import get_allowed_kwarg_only_overrides
|
||||
@@ -32,12 +31,14 @@ if TYPE_CHECKING:
|
||||
from transformers.processing_utils import ProcessorMixin
|
||||
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.renderers import TokenizeParams
|
||||
else:
|
||||
PretrainedConfig = object
|
||||
BatchFeature = object
|
||||
ProcessorMixin = object
|
||||
|
||||
ModelConfig = object
|
||||
TokenizeParams = object
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -339,6 +340,8 @@ class BaseProcessingInfo:
|
||||
|
||||
def get_default_tok_params(self) -> TokenizeParams:
|
||||
"""Construct the default parameters for tokenization."""
|
||||
from vllm.renderers import TokenizeParams
|
||||
|
||||
model_config = self.ctx.model_config
|
||||
encoder_config = model_config.encoder_config or {}
|
||||
|
||||
@@ -451,8 +454,7 @@ class BaseProcessingInfo:
|
||||
validate: bool = True,
|
||||
) -> MultiModalDataItems:
|
||||
"""
|
||||
Normalize
|
||||
[`MultiModalDataDict`][vllm.multimodal.inputs.MultiModalDataDict]
|
||||
Normalize [`MultiModalDataDict`][vllm.inputs.MultiModalDataDict]
|
||||
to [`MultiModalDataItems`][vllm.multimodal.parse.MultiModalDataItems]
|
||||
before passing them to
|
||||
[`_get_hf_mm_data`][vllm.multimodal.processing.BaseMultiModalProcessor._get_hf_mm_data].
|
||||
|
||||
@@ -14,9 +14,9 @@ from vllm.config.multimodal import (
|
||||
ImageDummyOptions,
|
||||
VideoDummyOptions,
|
||||
)
|
||||
from vllm.inputs import MultiModalDataDict
|
||||
from vllm.logger import init_logger
|
||||
|
||||
from ..inputs import MultiModalDataDict
|
||||
from .context import BaseProcessingInfo
|
||||
from .inputs import ProcessorInputs
|
||||
|
||||
|
||||
@@ -3,8 +3,9 @@
|
||||
from collections.abc import Mapping
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
from vllm.inputs import MultiModalHashes
|
||||
|
||||
from ..hasher import MultiModalHasher
|
||||
from ..inputs import MultiModalHashes
|
||||
from ..parse import MultiModalDataItems, MultiModalUUIDItems
|
||||
|
||||
|
||||
@@ -26,7 +27,7 @@ class ProcessorInputs:
|
||||
mm_uuid_items = self.mm_uuid_items or {}
|
||||
hf_processor_mm_kwargs = self.hf_processor_mm_kwargs
|
||||
|
||||
mm_hashes: MultiModalHashes = {}
|
||||
mm_hashes = dict[str, list[str]]()
|
||||
hasher = MultiModalHasher
|
||||
|
||||
for modality, data_items in mm_data_items.items():
|
||||
|
||||
@@ -6,34 +6,29 @@ from collections.abc import Callable, Generator, ItemsView, Iterable, Mapping, S
|
||||
from dataclasses import dataclass, field, replace
|
||||
from enum import Enum
|
||||
from functools import lru_cache
|
||||
from typing import (
|
||||
TYPE_CHECKING,
|
||||
Generic,
|
||||
NamedTuple,
|
||||
Protocol,
|
||||
TypeAlias,
|
||||
cast,
|
||||
)
|
||||
from typing import TYPE_CHECKING, Generic, NamedTuple, Protocol, TypeAlias, cast
|
||||
|
||||
import regex as re
|
||||
import torch
|
||||
from typing_extensions import TypeVar, assert_never
|
||||
|
||||
from vllm.inputs import (
|
||||
MultiModalEncDecInput,
|
||||
MultiModalHashes,
|
||||
MultiModalInput,
|
||||
mm_enc_dec_input,
|
||||
mm_input,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
from vllm.utils.collection_utils import flatten_2d_lists, full_groupby
|
||||
|
||||
from ..inputs import (
|
||||
MultiModalEncDecInputs,
|
||||
MultiModalFieldConfig,
|
||||
MultiModalHashes,
|
||||
MultiModalInputs,
|
||||
MultiModalKwargsItem,
|
||||
MultiModalKwargsItems,
|
||||
MultiModalKwargsOptionalItems,
|
||||
PlaceholderRange,
|
||||
mm_enc_dec_inputs,
|
||||
mm_inputs,
|
||||
)
|
||||
from ..parse import (
|
||||
DictEmbeddingItems,
|
||||
@@ -994,7 +989,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
mm_items: MultiModalDataItems,
|
||||
mm_uuid_items: MultiModalUUIDItems | None = None,
|
||||
hf_processor_mm_kwargs: Mapping[str, object] | None = None,
|
||||
) -> MultiModalInputs:
|
||||
) -> MultiModalInput:
|
||||
processor_inputs = ProcessorInputs(
|
||||
prompt,
|
||||
mm_items,
|
||||
@@ -1638,7 +1633,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
self,
|
||||
inputs: ProcessorInputs,
|
||||
timing_ctx: TimingContext,
|
||||
) -> MultiModalInputs:
|
||||
) -> MultiModalInput:
|
||||
"""
|
||||
Process multi-modal inputs to be used in vLLM.
|
||||
|
||||
@@ -1673,7 +1668,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
for modality, placeholders in mm_placeholders.items()
|
||||
}
|
||||
|
||||
return mm_inputs(
|
||||
return mm_input(
|
||||
prompt_token_ids=prompt_ids,
|
||||
mm_kwargs=mm_info.kwargs,
|
||||
mm_hashes=mm_info.hashes,
|
||||
@@ -1708,7 +1703,7 @@ class EncDecMultiModalProcessor(BaseMultiModalProcessor[_I]):
|
||||
self,
|
||||
prompt: str | list[int],
|
||||
mm_items: MultiModalDataItems,
|
||||
encoder_inputs: MultiModalInputs,
|
||||
encoder_inputs: MultiModalInput,
|
||||
):
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
decoder_prompt_raw = self.create_decoder_prompt(prompt, mm_items)
|
||||
@@ -1721,7 +1716,7 @@ class EncDecMultiModalProcessor(BaseMultiModalProcessor[_I]):
|
||||
decoder_prompt_text = None
|
||||
decoder_prompt_ids = decoder_prompt_raw
|
||||
|
||||
return mm_enc_dec_inputs(
|
||||
return mm_enc_dec_input(
|
||||
encoder_inputs,
|
||||
decoder_prompt_ids,
|
||||
decoder_prompt=decoder_prompt_text,
|
||||
@@ -1731,7 +1726,7 @@ class EncDecMultiModalProcessor(BaseMultiModalProcessor[_I]):
|
||||
self,
|
||||
inputs: ProcessorInputs,
|
||||
timing_ctx: TimingContext,
|
||||
) -> MultiModalEncDecInputs:
|
||||
) -> MultiModalEncDecInput:
|
||||
"""
|
||||
Process multi-modal inputs to be used in vLLM.
|
||||
The main processing steps are modified to fit encoder-decoder model:
|
||||
|
||||
@@ -7,6 +7,7 @@ from dataclasses import dataclass
|
||||
from multiprocessing.synchronize import Lock as LockType
|
||||
from typing import TYPE_CHECKING, Generic, Literal, Protocol, TypeVar, cast
|
||||
|
||||
from vllm.inputs import MultiModalInput
|
||||
from vllm.logger import init_logger
|
||||
from vllm.tokenizers import TokenizerLike, cached_tokenizer_from_config
|
||||
|
||||
@@ -19,7 +20,6 @@ from .cache import (
|
||||
ShmObjectStoreReceiverCache,
|
||||
ShmObjectStoreSenderCache,
|
||||
)
|
||||
from .inputs import MultiModalInputs
|
||||
from .processing import (
|
||||
BaseDummyInputsBuilder,
|
||||
BaseMultiModalProcessor,
|
||||
@@ -220,7 +220,7 @@ class MultiModalRegistry:
|
||||
*,
|
||||
cache: BaseMultiModalProcessorCache | None = None,
|
||||
processor: BaseMultiModalProcessor | None = None,
|
||||
) -> MultiModalInputs:
|
||||
) -> MultiModalInput:
|
||||
"""
|
||||
Create dummy data for profiling the memory usage of a model.
|
||||
|
||||
|
||||
@@ -12,6 +12,7 @@ import numpy.typing as npt
|
||||
from PIL import Image
|
||||
from typing_extensions import deprecated
|
||||
|
||||
from vllm.inputs import MultiModalPlaceholders
|
||||
from vllm.utils.import_utils import LazyLoader
|
||||
|
||||
from .hasher import MultiModalHasher
|
||||
@@ -19,7 +20,6 @@ from .inputs import (
|
||||
BatchedTensorInputs,
|
||||
MultiModalFieldElem,
|
||||
MultiModalKwargsItem,
|
||||
MultiModalPlaceholderDict,
|
||||
MultiModalSharedField,
|
||||
)
|
||||
from .media import AudioMediaIO, ImageMediaIO, MediaConnector, VideoMediaIO
|
||||
@@ -110,10 +110,10 @@ def encode_video_url(
|
||||
|
||||
|
||||
def argsort_mm_positions(
|
||||
mm_positions: MultiModalPlaceholderDict,
|
||||
mm_positions: MultiModalPlaceholders,
|
||||
) -> list[tuple[str, int]]:
|
||||
"""
|
||||
Given a `MultiModalPlaceholderDict`, output a sequence of keys to
|
||||
Given a `MultiModalPlaceholders`, output a sequence of keys to
|
||||
sort the dictionary by `offset` (starting index in the input sequence)
|
||||
in ascending order.
|
||||
|
||||
|
||||
Reference in New Issue
Block a user