Add pydocstyle to the ruff rules (#52136)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Signed-off-by: Harry Mellor <hej.mellor@gmail.com>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Harry Mellor
2026-09-16 16:44:29 +00:00
committed by GitHub
co-authored by Claude Opus 5
parent 804e5377aa
commit 8c1557a79c
2002 changed files with 15024 additions and 14269 deletions
+7
View File
@@ -45,6 +45,7 @@ def get_audio_duration(*, y: npt.NDArray[np.floating], sr: float = 22050) -> flo
Returns:
Duration of the audio in seconds.
"""
n_samples = y.shape[-1]
return float(n_samples) / sr
@@ -71,6 +72,7 @@ class AudioSpec:
(no normalization). 1 = mono, 2 = stereo, etc.
channel_reduction: Method to reduce channels when input has more
channels than target. Only used when reducing channels.
"""
target_channels: int | None = 1
@@ -119,6 +121,7 @@ def normalize_audio(
Raises:
ValueError: If audio has unsupported dimensions or channel expansion
is requested (e.g., mono to stereo).
"""
if not spec.needs_normalization:
return audio
@@ -195,6 +198,7 @@ def resample_audio_pyav(
Returns:
Resampled audio with the same shape as the input (1D → 1D, 2D → 2D).
"""
orig_sr_int = int(round(orig_sr))
target_sr_int = int(round(target_sr))
@@ -312,6 +316,7 @@ def resample_audio_torchaudio(
Returns:
Resampled audio with the same shape as the input (1D → 1D, 2D → 2D).
"""
orig_sr_int = int(round(orig_sr))
target_sr_int = int(round(target_sr))
@@ -431,6 +436,7 @@ def split_audio(
... )
>>> len(chunks)
3
"""
if audio_data.ndim > 1:
raise ValueError(
@@ -500,6 +506,7 @@ def find_split_point(
... )
>>> 16000 <= split_idx <= 17600
True
"""
segment = wav[start_idx:end_idx]
+32 -36
View File
@@ -63,12 +63,12 @@ class MultiModalCacheMissError(RuntimeError):
class MultiModalProcessorCacheItem:
"""
The data to store inside `MultiModalProcessorOnlyCache`.
"""The data to store inside `MultiModalProcessorOnlyCache`.
Args:
item: The processed tensor data corresponding to a multi-modal item.
prompt_updates: The prompt updates corresponding to `item`.
"""
def __init__(
@@ -83,8 +83,7 @@ class MultiModalProcessorCacheItem:
class MultiModalProcessorCacheItemMetadata:
"""
The metadata to store inside `MultiModalProcessorSenderCache`.
"""The metadata to store inside `MultiModalProcessorSenderCache`.
Args:
item: The processed tensor data corresponding to a multi-modal item.
@@ -94,6 +93,7 @@ class MultiModalProcessorCacheItemMetadata:
prompt_updates: The prompt updates corresponding to `item`.
This needs to stay on P0 because for some models, they are
dependent on the processed tensor data (cached on P1).
"""
def __init__(
@@ -163,8 +163,7 @@ class MultiModalCache:
@classmethod
def get_item_complexity(cls, value: MultiModalCacheValue) -> int:
"""
Get the number of leaf elements in a multi-modal cache value.
"""Get the number of leaf elements in a multi-modal cache value.
This provides a measure of structural complexity that can be useful
for debugging cache performance and understanding data patterns.
@@ -174,6 +173,7 @@ class MultiModalCache:
Returns:
The number of leaf elements in the nested structure.
"""
return json_count_leaves(value)
@@ -196,8 +196,7 @@ _O = TypeVar("_O", covariant=True)
class BaseMultiModalCache(ABC, Generic[_I, _O]):
"""
Abstract base class to read/write multi-modal items from cache.
"""Abstract base class to read/write multi-modal items from cache.
The idea of multi-modal caching is based on having a client and server
where the client executes in the frontend process (=P0) and
@@ -226,8 +225,7 @@ class BaseMultiModalCache(ABC, Generic[_I, _O]):
mm_item: _I,
mm_hash: str,
) -> _O:
"""
Possibly update a multi-modal item based on whether it is
"""Possibly update a multi-modal item based on whether it is
in the underlying cache.
This update is done out-of-place and updates the cache eviction order.
@@ -238,6 +236,7 @@ class BaseMultiModalCache(ABC, Generic[_I, _O]):
Returns:
The update multi-modal item.
"""
raise NotImplementedError
@@ -246,8 +245,7 @@ class BaseMultiModalCache(ABC, Generic[_I, _O]):
mm_items: Sequence[_I],
mm_hashes: list[str],
) -> list[_O]:
"""
Possibly update a sequence of multi-modal items based on whether they
"""Possibly update a sequence of multi-modal items based on whether they
are in the underlying cache.
This update is done out-of-place and updates the cache eviction order.
@@ -258,6 +256,7 @@ class BaseMultiModalCache(ABC, Generic[_I, _O]):
Returns:
A new list of updated multi-modal items.
"""
assert len(mm_items) == len(mm_hashes)
@@ -289,6 +288,7 @@ class BaseMultiModalCache(ABC, Generic[_I, _O]):
Returns:
`True` if the item was cached, otherwise `False`.
"""
if cache.put_if_fits(key, value):
return True
@@ -325,8 +325,7 @@ class BaseMultiModalProcessorCache(
@abstractmethod
def is_cached_item(self, mm_hash: str) -> bool:
"""
Check whether a multi-modal item is
"""Check whether a multi-modal item is
in the underlying cache.
This **DOES NOT** update the cache eviction order.
@@ -336,12 +335,12 @@ class BaseMultiModalProcessorCache(
Returns:
`True` if the item is cached, otherwise `False`.
"""
raise NotImplementedError
def is_cached(self, mm_hashes: list[str]) -> list[bool]:
"""
Check whether a sequence of multi-modal items are
"""Check whether a sequence of multi-modal items are
in the underlying cache.
This **DOES NOT** update the cache eviction order.
@@ -351,6 +350,7 @@ class BaseMultiModalProcessorCache(
Returns:
For each item, `True` if the item is cached, otherwise `False`.
"""
return [self.is_cached_item(mm_hash) for mm_hash in mm_hashes]
@@ -367,31 +367,30 @@ class BaseMultiModalProcessorCache(
@abstractmethod
def touch_sender_cache_item(self, mm_hash: str) -> None:
"""
Update the cache eviction order for a multi-modal item.
"""Update the cache eviction order for a multi-modal item.
This is used to touch the item in the cache without changing
its value.
Args:
mm_hash: The hash of the multi-modal item.
"""
raise NotImplementedError
@abstractmethod
def make_stats(self, *, delta: bool = False) -> CacheInfo:
"""
Get (and reset) the multi-modal cache stats.
"""Get (and reset) the multi-modal cache stats.
Returns:
The current multi-modal caching stats.
"""
raise NotImplementedError
class MultiModalProcessorOnlyCache(BaseMultiModalProcessorCache):
"""
The cache which is used on P0 when IPC caching is disabled.
"""The cache which is used on P0 when IPC caching is disabled.
How to update each item:
@@ -442,8 +441,7 @@ class MultiModalProcessorOnlyCache(BaseMultiModalProcessorCache):
class MultiModalProcessorSenderCache(BaseMultiModalProcessorCache):
"""
The cache which is used on P0 when IPC caching is enabled.
"""The cache which is used on P0 when IPC caching is enabled.
How to update each item:
@@ -507,8 +505,7 @@ class MultiModalProcessorSenderCache(BaseMultiModalProcessorCache):
class ShmObjectStoreSenderCache(BaseMultiModalProcessorCache):
"""
The cache which is used on P0 when IPC caching is enabled.
"""The cache which is used on P0 when IPC caching is enabled.
How to update each item:
@@ -609,7 +606,8 @@ class ShmObjectStoreSenderCache(BaseMultiModalProcessorCache):
@override
def touch_sender_cache_item(self, mm_hash: str) -> None:
"""Touch the item in shared memory cache to prevent eviction.
Increments writer_flag on sender side."""
Increments writer_flag on sender side.
"""
self._shm_cache.touch(mm_hash)
@override
@@ -662,8 +660,7 @@ class BaseMultiModalReceiverCache(
self,
mm_features: list["MultiModalFeatureSpec"],
) -> list["MultiModalFeatureSpec"]:
"""
Update multimodal features with cached encoder outputs.
"""Update multimodal features with cached encoder outputs.
Touch all identifier at first before update to avoid
item in updated list evict during update.
@@ -695,8 +692,7 @@ class BaseMultiModalReceiverCache(
mm_hash: str,
mm_item: MultiModalKwargsItem | None = None,
) -> None:
"""
Update the cache eviction order for a multi-modal item.
"""Update the cache eviction order for a multi-modal item.
This is used to touch the item in the cache without changing
its value.
@@ -705,13 +701,13 @@ class BaseMultiModalReceiverCache(
mm_hash: The hash of the multi-modal item.
mm_item: The multi-modal item itself. This is optional and
may not be needed by some cache implementations.
"""
raise NotImplementedError
class MultiModalReceiverCache(BaseMultiModalReceiverCache):
"""
The cache which is used on P1 when IPC caching is enabled.
"""The cache which is used on P1 when IPC caching is enabled.
How to update each item:
@@ -762,8 +758,7 @@ class MultiModalReceiverCache(BaseMultiModalReceiverCache):
class ShmObjectStoreReceiverCache(BaseMultiModalReceiverCache):
"""
The cache which is used on P1 Worker Process when IPC caching is enabled.
"""The cache which is used on P1 Worker Process when IPC caching is enabled.
How to update each item:
@@ -827,7 +822,8 @@ class ShmObjectStoreReceiverCache(BaseMultiModalReceiverCache):
mm_item: MultiModalKwargsItem | None = None,
) -> None:
"""Touch the item in shared memory cache to prevent eviction.
Increments reader_count on receiver side."""
Increments reader_count on receiver side.
"""
assert mm_item is not None
if "address" in mm_item:
address = cast(int, mm_item["address"].data)
+1 -2
View File
@@ -19,8 +19,7 @@ def get_mm_max_toks_per_item(
processor: BaseMultiModalProcessor,
mm_counts: Mapping[str, int],
) -> Mapping[str, int]:
"""
Get the maximum number of tokens per data item from each modality based
"""Get the maximum number of tokens per data item from each modality based
on underlying model configuration.
"""
max_tokens_per_item = processor.info.get_mm_max_tokens_per_item(
+1
View File
@@ -183,6 +183,7 @@ def reserve_mm_ipc_gpu_memory(
Raises:
ValueError: If the reservation leaves no memory for the KV cache.
"""
if mm_config is None:
return available_kv_cache_memory_bytes
+2 -3
View File
@@ -23,8 +23,7 @@ logger = init_logger(__name__)
def _get_hasher_factory(
algorithm: MMHasherAlgorithm,
) -> Callable[[], "hashlib._Hash"]:
"""
Get the hasher factory based on the configured algorithm.
"""Get the hasher factory based on the configured algorithm.
Args:
algorithm: Hash algorithm name (blake3, sha256, or sha512)
@@ -33,8 +32,8 @@ def _get_hasher_factory(
Supports blake3 (default), sha256, and sha512 for FIPS compliance.
See: https://github.com/vllm-project/vllm/issues/18334
"""
"""
if algorithm == "blake3":
from blake3 import blake3
+2 -1
View File
@@ -38,7 +38,8 @@ def rgba_to_rgb(
def _has_transparency(image: Image.Image) -> bool:
"""Detect whether an image carries transparency data (RGBA, LA, PA,
or tRNS chunk in P/L/RGB PNGs)."""
or tRNS chunk in P/L/RGB PNGs).
"""
if image.mode in ("RGBA", "LA", "PA"):
return True
return "transparency" in getattr(image, "info", {})
+30 -47
View File
@@ -120,11 +120,9 @@ VisionChunk: TypeAlias = VisionChunkImage | VisionChunkVideo
@dataclass(frozen=True)
class PlaceholderRange:
"""
Placeholder location information for multi-modal data.
"""Placeholder location information for multi-modal data.
Example:
Prompt: `AAAA BBBB What is in these images?`
Images A and B will have:
@@ -133,6 +131,7 @@ class PlaceholderRange:
A: PlaceholderRange(offset=0, length=4)
B: PlaceholderRange(offset=5, length=4)
```
"""
offset: int
@@ -161,8 +160,7 @@ class PlaceholderRange:
def get_embeds_indices_in_range(
self, start_idx: int, end_idx: int
) -> tuple[int, int]:
"""
Returns the starting and ending indices of the embeddings of encoder outputs
"""Returns the starting and ending indices of the embeddings of encoder outputs
in the range of [start_idx, end_idx) in the placeholders.
For example, given:
@@ -190,6 +188,7 @@ class PlaceholderRange:
A tuple `(start, end)` representing the start and end
indices (inclusive) of the embedded region.
Returns full placeholder range if `is_embed` is `None`.
"""
if self.is_embed is None:
return [(self.offset, self.offset + self.length - 1)]
@@ -234,8 +233,7 @@ def nested_tensors_equal(
b: NestedTensors,
check_dtype: bool = True,
) -> bool:
"""
Equality check between
"""Equality check between
[`NestedTensors`][vllm.multimodal.inputs.NestedTensors] objects.
If `check_dtype` is `True`, the tensors must have the same dtype.
@@ -297,7 +295,8 @@ def _is_dense(t: torch.Tensor) -> bool:
"""Whether `t`'s elements fill a contiguous storage range, possibly
permuted (e.g. a transposed matrix). Such layouts can be copied with
pitched cudaMemcpy2D/3DAsync directly from pinned memory, whereas
gapped layouts (e.g. strided slices) stage through a pageable temp."""
gapped layouts (e.g. strided slices) stage through a pageable temp.
"""
if not t.is_contiguous():
expected_stride = 1
for stride, size in sorted(zip(t.stride(), t.shape)):
@@ -336,8 +335,7 @@ A dictionary containing nested tensors which have been batched via
def batched_tensors_equal(a: BatchedTensorInputs, b: BatchedTensorInputs) -> bool:
"""
Equality check between
"""Equality check between
[`BatchedTensorInputs`][vllm.multimodal.inputs.BatchedTensorInputs] objects.
"""
return all(k in b and nested_tensors_equal(a[k], b[k]) for k in a)
@@ -345,8 +343,7 @@ def batched_tensors_equal(a: BatchedTensorInputs, b: BatchedTensorInputs) -> boo
@dataclass
class MultiModalFeatureSpec:
"""
Represents a single multimodal input with its processed data and metadata.
"""Represents a single multimodal input with its processed data and metadata.
Used to track multimodal data through processing and caching.
A request containing multiple multimodal items will have one
@@ -392,8 +389,7 @@ class MultiModalFeatureSpec:
@dataclass
class MultiModalFieldElem:
"""
Represents a processed keyword argument to pass to a model for a
"""Represents a processed keyword argument to pass to a model for a
[`MultiModalKwargsItem`][vllm.multimodal.inputs.MultiModalKwargsItem].
"""
@@ -429,8 +425,7 @@ class MultiModalFieldElem:
@dataclass(frozen=True, kw_only=True)
class BaseMultiModalField(ABC):
"""
Defines how to interpret tensor data belonging to a keyword argument for
"""Defines how to interpret tensor data belonging to a keyword argument for
[`MultiModalKwargsItems`][vllm.multimodal.inputs.MultiModalKwargsItems],
and vice versa.
"""
@@ -458,8 +453,7 @@ class BaseMultiModalField(ABC):
key: str,
data: NestedTensors,
) -> Sequence[MultiModalFieldElem]:
"""
Construct
"""Construct
[`MultiModalFieldElem`][vllm.multimodal.inputs.MultiModalFieldElem]
instances to represent the provided data.
@@ -481,8 +475,7 @@ class BaseMultiModalField(ABC):
device: torch.types.Device = None,
pin_memory: bool = False,
) -> NestedTensors:
"""
Merge the data from multiple instances of
"""Merge the data from multiple instances of
[`MultiModalFieldElem`][vllm.multimodal.inputs.MultiModalFieldElem].
This is the inverse of
@@ -511,9 +504,8 @@ class BaseMultiModalField(ABC):
@dataclass(frozen=True, kw_only=True)
class MultiModalBatchedField(BaseMultiModalField):
"""
Info:
[`MultiModalFieldConfig.batched`][vllm.multimodal.inputs.MultiModalFieldConfig.batched]
"""Info:
[`MultiModalFieldConfig.batched`][vllm.multimodal.inputs.MultiModalFieldConfig.batched]
"""
def build_elems(
@@ -547,10 +539,9 @@ class MultiModalBatchedField(BaseMultiModalField):
@dataclass(frozen=True, kw_only=True)
class MultiModalFlatField(BaseMultiModalField):
"""
Info:
[`MultiModalFieldConfig.flat`][vllm.multimodal.inputs.MultiModalFieldConfig.flat]
[`MultiModalFieldConfig.flat_from_sizes`][vllm.multimodal.inputs.MultiModalFieldConfig.flat_from_sizes]
"""Info:
[`MultiModalFieldConfig.flat`][vllm.multimodal.inputs.MultiModalFieldConfig.flat]
[`MultiModalFieldConfig.flat_from_sizes`][vllm.multimodal.inputs.MultiModalFieldConfig.flat_from_sizes]
"""
slices: Sequence[slice] | Sequence[Sequence[slice]]
@@ -637,9 +628,8 @@ class MultiModalFlatField(BaseMultiModalField):
@dataclass(frozen=True, kw_only=True)
class MultiModalSharedField(BaseMultiModalField):
"""
Info:
[`MultiModalFieldConfig.shared`][vllm.multimodal.inputs.MultiModalFieldConfig.shared]
"""Info:
[`MultiModalFieldConfig.shared`][vllm.multimodal.inputs.MultiModalFieldConfig.shared]
"""
batch_size: int
@@ -666,8 +656,7 @@ class MultiModalSharedField(BaseMultiModalField):
class MultiModalFieldConfig:
@staticmethod
def batched(modality: str, *, keep_on_cpu: bool = False):
"""
Defines a field where an element in the batch is obtained by
"""Defines a field where an element in the batch is obtained by
indexing into the first dimension of the underlying data.
Args:
@@ -676,7 +665,6 @@ class MultiModalFieldConfig:
keep_on_cpu: Whether to keep this field on the CPU for the model inputs.
Example:
```
Input:
Data: [[AAAA]
@@ -688,6 +676,7 @@ class MultiModalFieldConfig:
Element 2: [BBBB]
Element 3: [CCCC]
```
"""
return MultiModalFieldConfig(
field=MultiModalBatchedField(keep_on_cpu=keep_on_cpu),
@@ -702,8 +691,7 @@ class MultiModalFieldConfig:
*,
keep_on_cpu: bool = False,
):
"""
Defines a field where an element in the batch is obtained by
"""Defines a field where an element in the batch is obtained by
slicing along the first dimension of the underlying data.
Args:
@@ -716,7 +704,6 @@ class MultiModalFieldConfig:
keep_on_cpu: Whether to keep this field on the CPU for the model inputs.
Example:
```
Given:
slices: [slice(0, 3), slice(3, 7), slice(7, 9)]
@@ -746,6 +733,7 @@ class MultiModalFieldConfig:
Element 2: [[B],[B],[B],[B]]
Element 3: [[C],[C]]
```
"""
return MultiModalFieldConfig(
field=MultiModalFlatField(
@@ -764,8 +752,7 @@ class MultiModalFieldConfig:
*,
keep_on_cpu: bool = False,
):
"""
Defines a field where an element in the batch is obtained by
"""Defines a field where an element in the batch is obtained by
slicing along the first dimension of the underlying data.
Args:
@@ -777,7 +764,6 @@ class MultiModalFieldConfig:
keep_on_cpu: Whether to keep this field on the CPU for the model inputs.
Example:
```
Given:
size_per_item: [3, 4, 2]
@@ -807,8 +793,8 @@ class MultiModalFieldConfig:
Info:
[`MultiModalFieldConfig.flat`][vllm.multimodal.inputs.MultiModalFieldConfig.flat]
"""
"""
if size_per_item.ndim != 1:
raise ValueError(
"size_per_item should be a 1-D tensor, "
@@ -836,8 +822,7 @@ class MultiModalFieldConfig:
*,
keep_on_cpu: bool = False,
):
"""
Defines a field where an element in the batch is obtained by
"""Defines a field where an element in the batch is obtained by
taking the entirety of the underlying data.
This means that the data is the same for each element in the batch.
@@ -849,7 +834,6 @@ class MultiModalFieldConfig:
keep_on_cpu: Whether to keep this field on the CPU for the model inputs.
Example:
```
Given:
batch_size: 4
@@ -863,6 +847,7 @@ class MultiModalFieldConfig:
Element 3: [XYZ]
Element 4: [XYZ]
```
"""
return MultiModalFieldConfig(
field=MultiModalSharedField(
@@ -884,8 +869,7 @@ class MultiModalFieldConfig:
class MultiModalKwargsItem(UserDict[str, MultiModalFieldElem]):
"""
A dictionary of processed keyword arguments to pass to the model,
"""A dictionary of processed keyword arguments to pass to the model,
corresponding to a single item in
[`MultiModalDataItems`][vllm.multimodal.parse.MultiModalDataItems].
"""
@@ -912,8 +896,7 @@ _I = TypeVar(
class MultiModalKwargsItems(UserDict[str, Sequence[_I]]):
"""
A dictionary of processed multi-modal inputs by modality.
"""A dictionary of processed multi-modal inputs by modality.
For example, given a processor that processes
images into `pixel_values` and `image_grid_thw`,
+14 -1
View File
@@ -92,6 +92,10 @@ def load_audio_pyav(
Args:
path: A :class:`~io.BytesIO` buffer, a filesystem
:class:`~pathlib.Path`, or a string path.
sr: Target sample rate, or None to keep the native rate.
mono: Whether to average the channels down to mono.
max_decode_bytes: If set, abort decoding once this many bytes have
been read from the source.
max_duration_s: If set, abort decoding once the accumulated
sample count exceeds this many seconds of audio. Prevents
decompression-bomb attacks where a small compressed file
@@ -100,6 +104,7 @@ def load_audio_pyav(
Returns:
``(waveform, sample_rate)`` where *waveform* is a 1-D float32
NumPy array and *sample_rate* is the native sample rate in Hz.
"""
try:
container = av.open(path)
@@ -209,7 +214,7 @@ def load_audio_soundfile(
max_duration_s: float | None = None,
max_decode_bytes: int | None = None,
) -> tuple[np.ndarray, int]:
"""Load audio via soundfile"""
"""Load audio via soundfile."""
with soundfile.SoundFile(path) as f:
native_sr = f.samplerate
if max_duration_s is not None:
@@ -274,6 +279,7 @@ def load_audio_torchcodec(
``(waveform, sample_rate)`` where *waveform* is a float32 NumPy
array (1-D when ``mono=True``) and *sample_rate* is the output
sample rate in Hz.
"""
if AudioDecoder is None:
# Unify "torchcodec not installed" and "system ffmpeg missing" into
@@ -412,10 +418,17 @@ def load_audio(
"""Load audio using the selected decoding backend.
Args:
path: Audio file path or in-memory buffer.
sr: Target sample rate, or None to keep the native rate.
mono: Whether to downmix to a single channel.
max_duration_s: Reject audio longer than this many seconds.
max_decode_bytes: Reject audio that would decode to more than this
many bytes.
backend: One of ``AUDIO_BACKENDS``. ``None`` (default) selects
``"auto"``, which tries soundfile, then torchcodec, then PyAV;
the other values select a single
backend with no fallback.
"""
backend = backend or "auto"
if backend not in AUDIO_BACKENDS:
+2 -4
View File
@@ -14,8 +14,7 @@ _T = TypeVar("_T")
@dataclass
class MediaWithBytes(Generic[_T]):
"""
Wrapper that couples a media object with its original encoded bytes.
"""Wrapper that couples a media object with its original encoded bytes.
This ensures the raw bytes and media object remain synchronized,
preventing cache corruption from in-place modifications.
@@ -89,8 +88,7 @@ class MediaIO(ABC, Generic[_T]):
@abstractmethod
def load_base64(self, media_type: str, data: str) -> _T:
"""
List of media types:
"""List of media types:
https://www.iana.org/assignments/media-types/media-types.xhtml
"""
raise NotImplementedError
+23 -43
View File
@@ -73,6 +73,7 @@ def _wrap_media_fetch_error(
408/429, invalid URL)
Original exception for transient errors (5xx, 408, 429, network blips)
or other exceptions
"""
if isinstance(exc, VLLMValidationError):
return exc
@@ -162,16 +163,16 @@ class MediaConnector:
allowed_local_media_path: str = "",
allowed_media_domains: list[str] | None = None,
) -> None:
"""
Args:
media_io_kwargs: Additional args passed to process media
inputs, keyed by modalities. For example,
to set num_frames for video, set
`--media-io-kwargs '{"video":{"num_frames":40}}'`
connection: HTTP connection client to download media contents.
allowed_local_media_path: A local directory to load media files from.
allowed_media_domains: If set, only media URLs that belong to this
domain can be used for multi-modal inputs.
"""Args:
media_io_kwargs: Additional args passed to process media
inputs, keyed by modalities. For example,
to set num_frames for video, set
`--media-io-kwargs '{"video":{"num_frames":40}}'`
connection: HTTP connection client to download media contents.
allowed_local_media_path: A local directory to load media files from.
allowed_media_domains: If set, only media URLs that belong to this
domain can be used for multi-modal inputs.
"""
super().__init__()
@@ -468,9 +469,7 @@ class MediaConnector:
self,
audio_url: str,
) -> tuple[np.ndarray, int | float]:
"""
Load audio from a URL.
"""
"""Load audio from a URL."""
audio_io = AudioMediaIO(**self.media_io_kwargs.get("audio", {}))
return self.load_from_url(
@@ -483,9 +482,7 @@ class MediaConnector:
self,
audio_url: str,
) -> tuple[np.ndarray, int | float]:
"""
Asynchronously fetch audio from a URL.
"""
"""Asynchronously fetch audio from a URL."""
audio_io = AudioMediaIO(**self.media_io_kwargs.get("audio", {}))
return await self.load_from_url_async(
@@ -500,8 +497,7 @@ class MediaConnector:
*,
image_mode: str | None = "RGB",
) -> Image.Image:
"""
Load a PIL image from an HTTP or base64 data URL.
"""Load a PIL image from an HTTP or base64 data URL.
By default, the image is converted into RGB format. Set
`media_io_kwargs={"image": {"image_mode": None}}` to keep the
@@ -527,8 +523,7 @@ class MediaConnector:
*,
image_mode: str | None = "RGB",
) -> Image.Image:
"""
Asynchronously load a PIL image from an HTTP or base64 data URL.
"""Asynchronously load a PIL image from an HTTP or base64 data URL.
By default, the image is converted into RGB format. Set
`media_io_kwargs={"image": {"image_mode": None}}` to keep the
@@ -555,9 +550,7 @@ class MediaConnector:
image_mode: str | None = "RGB",
video_processor: str | None = None,
) -> MediaWithBytes[tuple[npt.NDArray, dict[str, Any]]]:
"""
Load video from an HTTP or base64 data URL.
"""
"""Load video from an HTTP or base64 data URL."""
image_io = ImageMediaIO(
**({"image_mode": image_mode} | self.media_io_kwargs.get("image", {}))
)
@@ -581,8 +574,7 @@ class MediaConnector:
image_mode: str | None = "RGB",
video_processor: str | None = None,
) -> MediaWithBytes[tuple[npt.NDArray, dict[str, Any]]]:
"""
Asynchronously load video from an HTTP or base64 data URL.
"""Asynchronously load video from an HTTP or base64 data URL.
By default, the image is converted into RGB format. Set
`media_io_kwargs={"image": {"image_mode": None}}` to keep the
@@ -608,9 +600,7 @@ class MediaConnector:
self,
data: str,
) -> torch.Tensor:
"""
Load image embedding from a URL.
"""
"""Load image embedding from a URL."""
image_embedding_io = ImageEmbeddingMediaIO()
return image_embedding_io.load_base64("", data)
@@ -619,9 +609,7 @@ class MediaConnector:
self,
data: str,
) -> torch.Tensor:
"""
Asynchronously load image embedding from a URL.
"""
"""Asynchronously load image embedding from a URL."""
image_embedding_io = ImageEmbeddingMediaIO()
loop = asyncio.get_running_loop()
@@ -633,9 +621,7 @@ class MediaConnector:
self,
data: str,
) -> torch.Tensor:
"""
Load audio embedding from a URL.
"""
"""Load audio embedding from a URL."""
audio_embedding_io = AudioEmbeddingMediaIO()
return audio_embedding_io.load_base64("", data)
@@ -644,9 +630,7 @@ class MediaConnector:
self,
data: str,
) -> torch.Tensor:
"""
Asynchronously load audio embedding from a URL.
"""
"""Asynchronously load audio embedding from a URL."""
audio_embedding_io = AudioEmbeddingMediaIO()
loop = asyncio.get_running_loop()
@@ -658,9 +642,7 @@ class MediaConnector:
self,
data: str,
) -> torch.Tensor:
"""
Load video embedding from a URL.
"""
"""Load video embedding from a URL."""
video_embedding_io = VideoEmbeddingMediaIO()
return video_embedding_io.load_base64("", data)
@@ -669,9 +651,7 @@ class MediaConnector:
self,
data: str,
) -> torch.Tensor:
"""
Asynchronously load video embedding from a URL.
"""
"""Asynchronously load video embedding from a URL."""
video_embedding_io = VideoEmbeddingMediaIO()
loop = asyncio.get_running_loop()
+22 -28
View File
@@ -58,8 +58,7 @@ else:
class ModalityDataItems(ABC, Generic[_T, _I]):
"""
Represents data items for a modality in
"""Represents data items for a modality in
[`MultiModalDataItems`][vllm.multimodal.parse.MultiModalDataItems].
"""
@@ -151,6 +150,7 @@ def validate_embedding_ndim(
tensor: The tensor to validate.
modality: The modality name for error messages (e.g., "image", "audio").
index: Optional index for list items, included in error messages.
"""
if tensor.ndim < 2 or tensor.ndim > 3:
idx_str = f" [{index}]" if index is not None else ""
@@ -164,8 +164,7 @@ def validate_embedding_ndim(
class EmbeddingItems(
ModalityDataItems[torch.Tensor | list[torch.Tensor], torch.Tensor]
):
"""
Base class for data items that are expressed as a batched embedding tensor,
"""Base class for data items that are expressed as a batched embedding tensor,
or a list of embedding tensors (one per item).
"""
@@ -251,8 +250,7 @@ class EmbeddingItems(
class DictEmbeddingItems(
ModalityDataItems[Mapping[str, torch.Tensor], Mapping[str, torch.Tensor]]
):
"""
Base class for data items that are expressed as a dictionary of tensors.
"""Base class for data items that are expressed as a dictionary of tensors.
Usually, the dictionary keys correspond to the outputs of HF processor.
"""
@@ -268,18 +266,18 @@ class DictEmbeddingItems(
],
optional_fields: Set[str] = frozenset(),
) -> None:
"""
Args:
data: The dictionary of tensors for this modality.
modality: The modality these items belong to.
required_fields: Fields `data` must contain.
fields_factory: Builds the field config from the data.
optional_fields: Fields `data` may omit. Which fields these are is
the caller's decision -- see
`MultiModalDataParser.embedding_field_sets`, where a deployment
that receives embeddings through an EC connector makes the
embeddings optional. They still need a field config, since they
are used whenever they *are* supplied.
"""Args:
data: The dictionary of tensors for this modality.
modality: The modality these items belong to.
required_fields: Fields `data` must contain.
fields_factory: Builds the field config from the data.
optional_fields: Fields `data` may omit. Which fields these are is
the caller's decision -- see
`MultiModalDataParser.embedding_field_sets`, where a deployment
that receives embeddings through an EC connector makes the
embeddings optional. They still need a field config, since they
are used whenever they *are* supplied.
"""
from transformers.feature_extraction_utils import BatchFeature
@@ -471,14 +469,12 @@ _D = TypeVar("_D", bound=ModalityDataItems[Any, Any])
class MultiModalDataItems(UserDict[str, ModalityDataItems[Any, Any]]):
"""
A normalized [`MultiModalDataDict`][vllm.inputs.MultiModalDataDict]
"""A normalized [`MultiModalDataDict`][vllm.inputs.MultiModalDataDict]
such that each entry corresponds to a list.
"""
def select(self, modalities: Set[str]):
"""
Construct a new `MultiModalDataItems` instance containing only the
"""Construct a new `MultiModalDataItems` instance containing only the
selected modalities.
"""
return MultiModalDataItems(
@@ -486,8 +482,7 @@ class MultiModalDataItems(UserDict[str, ModalityDataItems[Any, Any]]):
)
def get_count(self, modality: str, *, strict: bool = True) -> int:
"""
Get the number of data items belonging to a modality.
"""Get the number of data items belonging to a modality.
If `strict=False`, return `0` instead of raising [`KeyError`][]
even if the modality is not found.
@@ -513,8 +508,7 @@ class MultiModalDataItems(UserDict[str, ModalityDataItems[Any, Any]]):
modality: str,
typ: type[_D] | tuple[type[_D], ...],
) -> _D:
"""
Get the data items belonging to a modality,
"""Get the data items belonging to a modality,
requiring that they belong to a certain type.
"""
if modality not in self:
@@ -541,8 +535,7 @@ ModalityDataParser: TypeAlias = Callable[
class MultiModalDataParser:
"""
Parses [`MultiModalDataDict`][vllm.inputs.MultiModalDataDict]
"""Parses [`MultiModalDataDict`][vllm.inputs.MultiModalDataDict]
into [`MultiModalDataItems`][vllm.multimodal.parse.MultiModalDataItems].
Args:
@@ -561,6 +554,7 @@ class MultiModalDataParser:
allow_missing_mm_embeddings (bool): Whether pre-computed embedding
tensors may be absent from the request on a disaggregated consumer.
Derived by `BaseProcessingInfo.allow_missing_mm_embeddings`.
"""
embedding_fields: Mapping[str, Mapping[str, EmbeddingFieldRole]] = {}
+19 -29
View File
@@ -70,6 +70,7 @@ def overlay_modality_mm_kwargs(
Returns:
A new dict with the modality-scoped keys overlaid when present.
"""
merged = dict(kwargs)
if modality is None:
@@ -128,8 +129,7 @@ _P = TypeVar("_P", bound=ProcessorMixin, default=ProcessorMixin)
@dataclass(frozen=True)
class InputProcessingContext:
"""
Contains information about the model which may be used to
"""Contains information about the model which may be used to
modify the inputs.
"""
@@ -162,13 +162,13 @@ class InputProcessingContext:
typ: type[Any] | tuple[type[Any], ...] | None = None,
/,
) -> Any:
"""
Get the HuggingFace configuration
"""Get the HuggingFace configuration
(`transformers.PretrainedConfig`) of the model,
additionally checking its type.
Raises:
TypeError: If the configuration is not of the specified type.
"""
if typ is None:
from transformers.configuration_utils import PretrainedConfig
@@ -186,17 +186,15 @@ class InputProcessingContext:
return hf_config
def get_hf_image_processor_config(self) -> dict[str, Any]:
"""
Get the HuggingFace image processor configuration of the model.
"""
"""Get the HuggingFace image processor configuration of the model."""
return self.model_config.hf_image_processor_config
def get_mm_config(self):
"""
Get the multimodal config of the model.
"""Get the multimodal config of the model.
Raises:
RuntimeError: If the model is not a multimodal model.
"""
mm_config = self.model_config.multimodal_config
if mm_config is None:
@@ -221,13 +219,13 @@ class InputProcessingContext:
/,
**kwargs: object,
) -> Any:
"""
Get the HuggingFace processor
"""Get the HuggingFace processor
(`transformers.ProcessorMixin`) of the model,
additionally checking its type.
Raises:
TypeError: If the processor is not of the specified type.
"""
if typ is None:
from transformers.processing_utils import ProcessorMixin
@@ -254,8 +252,7 @@ class InputProcessingContext:
/,
**kwargs: object,
) -> _T:
"""
Initialize a HuggingFace-like processor class, merging the
"""Initialize a HuggingFace-like processor class, merging the
keyword arguments with those in the model's configuration.
"""
merged_kwargs = self.get_merged_mm_kwargs(kwargs)
@@ -317,8 +314,7 @@ class InputProcessingContext:
data: Mapping[str, object],
kwargs: Mapping[str, object] = {},
) -> BatchFeature:
"""
Call `hf_processor` on the prompt `data`
"""Call `hf_processor` on the prompt `data`
(text, image, audio...) with configurable options `kwargs`.
"""
assert callable(hf_processor)
@@ -385,8 +381,7 @@ class BaseProcessingInfo:
return self.ctx.get_hf_config()
def get_hf_processor(self, **kwargs: object) -> ProcessorMixin:
"""
Subclasses can override this method to handle
"""Subclasses can override this method to handle
specific kwargs from model config or user inputs.
"""
return self.ctx.get_hf_processor(**kwargs)
@@ -409,8 +404,7 @@ class BaseProcessingInfo:
return self.get_default_tok_params()
def _get_expected_hidden_size(self) -> int | None:
"""
Get expected hidden size for embedding validation if `mm_embeds` are enabled.
"""Get expected hidden size for embedding validation if `mm_embeds` are enabled.
This validates hidden dimensions to prevent a vulnerability where embeddings
with correct `ndim` but wrong `shape` could cause crashes at inference time.
@@ -430,8 +424,7 @@ class BaseProcessingInfo:
return mm_config is not None and mm_config.allow_missing_mm_embeddings
def get_data_parser(self) -> MultiModalDataParser:
"""
Constructs a parser to preprocess multi-modal data items
"""Constructs a parser to preprocess multi-modal data items
before passing them to
[`_get_hf_mm_data`][vllm.multimodal.processing.BaseMultiModalProcessor._get_hf_mm_data].
@@ -453,8 +446,7 @@ class BaseProcessingInfo:
return False
def get_supported_mm_limits(self) -> Mapping[str, int | None]:
"""
Return the maximum supported number of items for each modality.
"""Return the maximum supported number of items for each modality.
A value of `None` means unlimited number of items.
@@ -486,8 +478,7 @@ class BaseProcessingInfo:
return allowed_limits
def validate_num_items(self, modality: str, num_items: int) -> None:
"""
Raise `ValueError` if the number of input items for the given modality
"""Raise `ValueError` if the number of input items for the given modality
is invalid.
"""
supported_limit = self.supported_mm_limits.get(modality, 0)
@@ -512,8 +503,7 @@ class BaseProcessingInfo:
*,
validate: bool = True,
) -> MultiModalDataItems:
"""
Normalize [`MultiModalDataDict`][vllm.inputs.MultiModalDataDict]
"""Normalize [`MultiModalDataDict`][vllm.inputs.MultiModalDataDict]
to [`MultiModalDataItems`][vllm.multimodal.parse.MultiModalDataItems]
before passing them to
[`_get_hf_mm_data`][vllm.multimodal.processing.BaseMultiModalProcessor._get_hf_mm_data].
@@ -546,8 +536,7 @@ class BaseProcessingInfo:
seq_len: int,
mm_counts: Mapping[str, int],
) -> Mapping[str, int] | None:
"""
Return the maximum number of tokens per item of for each modality.
"""Return the maximum number of tokens per item of for each modality.
When `None` (the default) is returned, vLLM will generate dummy inputs
(images/videos) at maximum possible sizes and process them to determine
@@ -564,5 +553,6 @@ class BaseProcessingInfo:
length and the maximum number of items of each modality allowed,
and agree with dummy inputs (images/videos) at maximum possible
sizes.
"""
return None
+6 -9
View File
@@ -26,8 +26,7 @@ logger = init_logger(__name__)
class BaseDummyInputsBuilder(ABC, Generic[_I]):
"""
Abstract base class that constructs the dummy data to profile
"""Abstract base class that constructs the dummy data to profile
multi-modal models.
"""
@@ -38,9 +37,7 @@ class BaseDummyInputsBuilder(ABC, Generic[_I]):
@abstractmethod
def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str:
"""
Build the text input corresponding to `mm_counts`.
"""
"""Build the text input corresponding to `mm_counts`."""
raise NotImplementedError
@abstractmethod
@@ -50,8 +47,7 @@ class BaseDummyInputsBuilder(ABC, Generic[_I]):
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
) -> MultiModalDataDict:
"""
Build the multimodal input which, after processing, results in
"""Build the multimodal input which, after processing, results in
the maximum possible number of placeholder tokens.
Args:
@@ -61,6 +57,7 @@ class BaseDummyInputsBuilder(ABC, Generic[_I]):
If None, use model defaults for backward compatibility.
If provided, models can use these to customize dummy
data generation.
"""
raise NotImplementedError
@@ -70,14 +67,14 @@ class BaseDummyInputsBuilder(ABC, Generic[_I]):
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
) -> ProcessorInputs:
"""
Build the input which, after processing, results in
"""Build the input which, after processing, results in
the maximum possible number of placeholder tokens.
Args:
seq_len: Sequence length
mm_counts: Count of items per modality
mm_options: Configurable options per modality (optional)
"""
dummy_text = self.get_dummy_text(mm_counts)
dummy_mm_data = self.get_dummy_mm_data(seq_len, mm_counts, mm_options)
+1 -2
View File
@@ -18,8 +18,7 @@ _HF_MODALITY_PROCESSOR_KWARGS = {
@dataclass
class ProcessorInputs:
"""
Represents the keyword arguments to
"""Represents the keyword arguments to
[`vllm.multimodal.processing.BaseMultiModalProcessor.apply`][].
"""
+29 -56
View File
@@ -104,8 +104,7 @@ class PromptIndex:
class PromptIndexTargets:
@staticmethod
def start() -> PromptIndex:
"""
Resolves to the start of the prompt (before the first token).
"""Resolves to the start of the prompt (before the first token).
This results in a match even if the prompt is empty.
"""
@@ -116,9 +115,7 @@ class PromptIndexTargets:
@staticmethod
def prefix(seq: list[int]) -> PromptIndex:
"""
Resolves to a location in the prompt after the given prefix.
"""
"""Resolves to a location in the prompt after the given prefix."""
def get_match_index(
prompt: list[int],
@@ -146,8 +143,7 @@ class PromptIndexTargets:
@staticmethod
def end() -> PromptIndex:
"""
Resolves to the end of the prompt (after the last token).
"""Resolves to the end of the prompt (after the last token).
This results in a match even if the prompt is empty.
"""
@@ -247,8 +243,7 @@ class UpdateMode(str, Enum):
@dataclass
class PromptUpdate(ABC):
"""
Defines how to update a prompt with placeholder tokens.
"""Defines how to update a prompt with placeholder tokens.
Note:
The target and content are token sequences. When converting text
@@ -256,6 +251,7 @@ class PromptUpdate(ABC):
`add_special_tokens=False` to match how the prompt itself is
tokenized. Otherwise, the updated prompt may contain duplicated
special tokens or fail to match the target.
"""
modality: str
@@ -294,8 +290,7 @@ class PromptUpdate(ABC):
return content
def resolve(self, item_idx: int) -> "ResolvedPromptUpdate":
"""
Given the index of the processed item within
"""Given the index of the processed item within
[`modality`][vllm.multimodal.processing.PromptUpdate.modality],
output a copy of this object with its lazy attributes resolved.
"""
@@ -310,11 +305,9 @@ class PromptUpdate(ABC):
@dataclass
class PromptInsertion(PromptUpdate):
"""
Defines how to insert placeholder tokens into a prompt.
"""Defines how to insert placeholder tokens into a prompt.
Example:
For each image, insert a number of `<image>` feature placeholders
equal to the feature size of the vision encoder after the `<s>` token:
@@ -355,6 +348,7 @@ class PromptInsertion(PromptUpdate):
insertion=[image_token_id] * image_feature_size,
)
```
"""
insertion: PromptUpdateContent = field(repr=False)
@@ -379,11 +373,9 @@ class PromptInsertion(PromptUpdate):
@dataclass
class PromptReplacement(PromptUpdate):
"""
Defines how to replace portions of an input prompt with placeholder tokens.
"""Defines how to replace portions of an input prompt with placeholder tokens.
Example:
For each image, replace one `<image>` input placeholder in the prompt
with a number of `<image>` feature placeholders
equal to the feature size of the vision encoder:
@@ -412,6 +404,7 @@ class PromptReplacement(PromptUpdate):
),
)
```
"""
replacement: PromptUpdateContent = field(repr=False)
@@ -447,8 +440,7 @@ _M = TypeVar("_M", bound=_HasModalityAttr | _HasModalityProp)
def full_groupby_modality(values: Iterable[_M]) -> ItemsView[str, list[_M]]:
"""
Convenience function to apply
"""Convenience function to apply
[`full_groupby`][vllm.utils.collection_utils.full_groupby]
based on modality.
"""
@@ -462,8 +454,7 @@ class PromptTargetMatch(NamedTuple):
@dataclass(frozen=True)
class ResolvedPromptUpdate:
"""
A [`PromptUpdate`][vllm.multimodal.processing.PromptUpdate] with its
"""A [`PromptUpdate`][vllm.multimodal.processing.PromptUpdate] with its
lazy attributes resolved.
"""
@@ -522,8 +513,7 @@ def iter_token_matches(
*,
start_idx: int = 0,
) -> Generator[_TokenMatch]:
"""
Yield each occurrence of `match_ids` in `token_ids`.
"""Yield each occurrence of `match_ids` in `token_ids`.
Note that empty matches are ignored.
"""
@@ -562,8 +552,7 @@ def replace_token_matches(
match_ids: list[int],
new_ids: list[int],
) -> list[int]:
"""
Replace each occurrence of `match_ids` in `token_ids`
"""Replace each occurrence of `match_ids` in `token_ids`
with `new_ids`.
Note that empty matches are ignored.
@@ -846,8 +835,7 @@ def apply_token_matches(
prompt: list[int],
mm_prompt_updates: "MultiModalPromptUpdates",
) -> tuple[list[int], "MultiModalPromptUpdatesApplyResult"]:
"""
Apply the updates in `mm_prompt_updates` to `prompt`.
"""Apply the updates in `mm_prompt_updates` to `prompt`.
Matches are exclusive even when multiple modalities share
the same placeholder tokens. In that case, the modality that
@@ -914,8 +902,7 @@ def _iter_placeholders(
prompt: list[int],
mm_prompt_updates: "MultiModalPromptUpdates",
) -> Iterable[PlaceholderFeaturesInfo]:
"""
Yield each set of placeholder tokens found in `prompt`.
"""Yield each set of placeholder tokens found in `prompt`.
Matches are exclusive even when multiple modalities share
the same placeholder tokens. In that case, the modality that
@@ -1028,8 +1015,7 @@ class MultiModalProcessingInfo(NamedTuple):
class BaseMultiModalProcessor(ABC, Generic[_I]):
"""
Abstract base class to process multi-modal inputs to be used in vLLM.
"""Abstract base class to process multi-modal inputs to be used in vLLM.
Not to be confused with `transformers.ProcessorMixin`.
"""
@@ -1097,8 +1083,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
hf_processor_mm_kwargs: Mapping[str, object],
out_mm_kwargs: MultiModalKwargsItems,
) -> Sequence[PromptUpdate]:
"""
Given the original multi-modal items for this modality
"""Given the original multi-modal items for this modality
and HF-processed data, output the updates to perform.
The information returned by this method is used to update token inputs
@@ -1167,8 +1152,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
return processor_data, passthrough_data
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str | None:
"""
Get the text to pass to the HF processor alongside the multi-modal
"""Get the text to pass to the HF processor alongside the multi-modal
data.
By default, no text is passed. If the HF processor requires that
@@ -1183,8 +1167,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
mm_data: Mapping[str, object],
hf_processor_mm_kwargs: Mapping[str, object],
) -> tuple[Mapping[str, object], Mapping[str, object]]:
"""
Pre-process the multi-modal data and HF processor keyword arguments
"""Pre-process the multi-modal data and HF processor keyword arguments
before they are passed to the HF processor.
By default, both are returned as-is. If the HF processor expects the
@@ -1201,8 +1184,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
hf_processor_mm_kwargs: Mapping[str, object],
processed_data: BatchFeature,
) -> BatchFeature:
"""
Post-process the output of the HF processor.
"""Post-process the output of the HF processor.
By default, the output is returned as-is. If you need to modify the
output of the HF processor before it is converted into multi-modal
@@ -1215,9 +1197,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
) -> BatchFeature:
"""
Apply the HF processor on the multi-modal data.
"""
"""Apply the HF processor on the multi-modal data."""
valid_mm_items = mm_items.select(
{k for k, c in mm_items.get_all_counts().items() if c > 0}
)
@@ -1250,8 +1230,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
)
def _postprocess_prompt(self, prompt: list[int]) -> list[int]:
"""
Post-process the prompt token IDs before locating or applying
"""Post-process the prompt token IDs before locating or applying
multi-modal prompt updates.
By default, the prompt is returned as-is. If the HF processor (or
@@ -1303,8 +1282,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
cached_update: ResolvedPromptUpdate,
new_item_idx: int,
) -> ResolvedPromptUpdate:
"""
Override this if other attributes of `ResolvedPromptUpdate`
"""Override this if other attributes of `ResolvedPromptUpdate`
also need to be recomputed after retrieving from the cache.
"""
return replace(cached_update, item_idx=new_item_idx)
@@ -1404,8 +1382,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
inputs: ProcessorInputs,
timing_ctx: TimingContext,
) -> MultiModalProcessingInfo:
"""
Apply the HF processor on the full prompt text,
"""Apply the HF processor on the full prompt text,
caching the results and reusing cached results.
"""
cache = self.cache
@@ -1522,8 +1499,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
*,
encode_segments_separately: bool = False,
) -> tuple[list[int], MultiModalPromptUpdatesApplyResult]:
"""
Apply multi-modal prompt updates in text space, then encode the
"""Apply multi-modal prompt updates in text space, then encode the
result.
A non-special-token target may be tokenized differently inside the
@@ -1716,8 +1692,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
inputs: ProcessorInputs,
timing_ctx: TimingContext,
) -> MultiModalInput:
"""
Process multi-modal inputs to be used in vLLM.
"""Process multi-modal inputs to be used in vLLM.
The main steps are:
@@ -1763,8 +1738,7 @@ class EncDecMultiModalProcessor(BaseMultiModalProcessor[_I]):
prompt: list[int],
mm_items: MultiModalDataItems,
) -> list[int]:
"""
Create input prompt for the encoder. HF processor will be applied on
"""Create input prompt for the encoder. HF processor will be applied on
this prompt during profiling and generation.
"""
raise NotImplementedError
@@ -1805,8 +1779,7 @@ class EncDecMultiModalProcessor(BaseMultiModalProcessor[_I]):
inputs: ProcessorInputs,
timing_ctx: TimingContext,
) -> MultiModalEncDecInput:
"""
Process multi-modal inputs to be used in vLLM.
"""Process multi-modal inputs to be used in vLLM.
The main processing steps are modified to fit encoder-decoder model:
1. Create encoder prompt from input prompt text.
2. Apply the HF processor on encoder prompt.
+8 -18
View File
@@ -45,8 +45,7 @@ _I_co = TypeVar("_I_co", bound=BaseProcessingInfo, covariant=True)
class ProcessingInfoFactory(Protocol[_I_co]):
"""
Constructs a
"""Constructs a
[`BaseMultiModalProcessor`][vllm.multimodal.processing.BaseMultiModalProcessor]
instance from the context.
"""
@@ -58,8 +57,7 @@ class ProcessingInfoFactory(Protocol[_I_co]):
class DummyInputsBuilderFactory(Protocol[_I]): # type: ignore[misc]
"""
Constructs a
"""Constructs a
[`BaseDummyInputsBuilder`][vllm.multimodal.processing.BaseDummyInputsBuilder]
instance from the context.
"""
@@ -68,8 +66,7 @@ class DummyInputsBuilderFactory(Protocol[_I]): # type: ignore[misc]
class MultiModalProcessorFactory(Protocol[_I]): # type: ignore[misc]
"""
Constructs a
"""Constructs a
[`BaseMultiModalProcessor`][vllm.multimodal.processing.BaseMultiModalProcessor]
instance from the context.
"""
@@ -101,13 +98,10 @@ class _ProcessorFactories(Generic[_I]):
class MultiModalRegistry:
"""
A registry that dispatches data processing according to the model.
"""
"""A registry that dispatches data processing according to the model."""
def supports_multimodal_inputs(self, model_config: "ModelConfig") -> bool:
"""
Checks if the model supports multimodal inputs.
"""Checks if the model supports multimodal inputs.
Returns True if the model is multimodal with any non-zero supported
modalities, otherwise returns False, effectively running in
text-only mode.
@@ -158,8 +152,7 @@ class MultiModalRegistry:
info: ProcessingInfoFactory[_I],
dummy_inputs: DummyInputsBuilderFactory[_I],
):
"""
Register a multi-modal processor to a model class. The processor
"""Register a multi-modal processor to a model class. The processor
is constructed lazily, hence a factory method should be passed.
When the model receives multi-modal data, the provided function is
@@ -227,9 +220,7 @@ class MultiModalRegistry:
tokenizer: TokenizerLike | None = None,
cache: BaseMultiModalProcessorCache | None = None,
) -> BaseMultiModalProcessor[BaseProcessingInfo]:
"""
Create a multi-modal processor for a specific model and tokenizer.
"""
"""Create a multi-modal processor for a specific model and tokenizer."""
if not model_config.is_multimodal_model:
model_name = model_config.served_model_name or model_config.model
raise ValueError(f"{model_name} is not a multimodal model")
@@ -250,8 +241,7 @@ class MultiModalRegistry:
processor: BaseMultiModalProcessor | None = None,
scheduler_config: "SchedulerConfig | None" = None,
) -> MultiModalInput:
"""
Create dummy data for profiling the memory usage of a model.
"""Create dummy data for profiling the memory usage of a model.
The model is identified by `model_config`.
"""
+16 -17
View File
@@ -67,8 +67,7 @@ def encode_image_base64(
image_mode: str | None = "RGB",
format: str = "PNG",
) -> str:
"""
Encode a pillow image to base64 format.
"""Encode a pillow image to base64 format.
By default, the image is converted into RGB format before being encoded.
Pass `image_mode=None` to keep the original image mode.
@@ -83,8 +82,7 @@ def encode_image_url(
image_mode: str | None = "RGB",
format: str = "PNG",
) -> str:
"""
Encode a pillow image as a data URL.
"""Encode a pillow image as a data URL.
By default, the image is converted into RGB format before being encoded.
Pass `image_mode=None` to keep the original image mode.
@@ -147,14 +145,14 @@ def get_mm_features_in_window(
def argsort_mm_positions(
mm_positions: MultiModalPlaceholders,
) -> list[tuple[str, int]]:
"""
Given a `MultiModalPlaceholders`, output a sequence of keys to
"""Given a `MultiModalPlaceholders`, output a sequence of keys to
sort the dictionary by `offset` (starting index in the input sequence)
in ascending order.
Returns:
A list of `(modality, idx)`, which can be used to access an item
by `mm_positions[modality][idx]`.
"""
flat_items = (
(modality, idx, item)
@@ -222,7 +220,8 @@ def strip_covered_mm_data(
so the workers never consume the payload fields. M-RoPE models keep
CPU-side metadata fields used to compute positions. SHM address items
are also kept so workers can balance the sender's reference count. The
scheduler-side ``Request`` keeps the full features."""
scheduler-side ``Request`` keeps the full features.
"""
if not mm_features or num_computed_tokens == 0:
return mm_features
@@ -253,8 +252,7 @@ def group_and_batch_mm_items(
device: torch.types.Device = None,
pin_memory: bool = False,
) -> Generator[tuple[int, BatchedTensorInputs]]:
"""
Group consecutive items (possibly from different requests) into batches.
"""Group consecutive items (possibly from different requests) into batches.
Items must be split across groups if any of the following occurs,
as the batch would otherwise be invalid:
@@ -270,6 +268,7 @@ def group_and_batch_mm_items(
A tuple `(num_items, grouped_kwargs)`, where:
- `kwargs` is a dictionary of keyword arguments to pass to the model;
- `num_items` is the corresponding number of items.
"""
start_idx = 0
for end_idx in range(1, len(items) + 1):
@@ -296,8 +295,7 @@ def group_and_batch_mm_kwargs(
device: torch.types.Device = None,
pin_memory: bool = False,
) -> Generator[tuple[str, int, BatchedTensorInputs], None, None]:
"""
Group consecutive items (possibly from different requests) into batches.
"""Group consecutive items (possibly from different requests) into batches.
Items must be split across groups if any of the following occurs,
as the batch would otherwise be invalid:
@@ -317,6 +315,7 @@ def group_and_batch_mm_kwargs(
- `modality` is the modality of the batch;
- `kwargs` is a dictionary of keyword arguments to pass to the model;
- `num_items` is the corresponding number of items.
"""
for modality, group in groupby(mm_kwargs, key=lambda x: x[0]):
items_lst = [item for _, item in group]
@@ -333,14 +332,14 @@ def fetch_audio(
audio_url: str,
audio_io_kwargs: dict[str, Any] | None = None,
) -> tuple[np.ndarray, int | float]:
"""
Args:
"""Args:
audio_url: URL of the audio file to fetch.
audio_io_kwargs: Additional kwargs passed to handle audio IO.
Warning:
This method has direct access to local files and is only intended
to be called by user code. Never call this from the online server!
"""
media_io_kwargs = None if not audio_io_kwargs else {"audio": audio_io_kwargs}
media_connector = MediaConnector(
@@ -354,14 +353,14 @@ def fetch_image(
image_url: str,
image_io_kwargs: dict[str, Any] | None = None,
) -> Image.Image:
"""
Args:
"""Args:
image_url: URL of the image file to fetch.
image_io_kwargs: Additional kwargs passed to handle image IO.
Warning:
This method has direct access to local files and is only intended
to be called by user code. Never call this from the online server!
"""
media_io_kwargs = None if not image_io_kwargs else {"image": image_io_kwargs}
media_connector = MediaConnector(
@@ -375,14 +374,14 @@ def fetch_video(
video_url: str,
video_io_kwargs: dict[str, Any] | None = None,
) -> MediaWithBytes[tuple[npt.NDArray, dict[str, Any]]]:
"""
Args:
"""Args:
video_url: URL of the video file to fetch.
video_io_kwargs: Additional kwargs passed to handle video IO.
Warning:
This method has direct access to local files and is only intended
to be called by user code. Never call this from the online server!
"""
media_io_kwargs = None if not video_io_kwargs else {"video": video_io_kwargs}
media_connector = MediaConnector(
+7 -6
View File
@@ -288,6 +288,7 @@ class VideoBackend(VideoLoader):
Returns:
Tuple of ``(frames, metadata_dict)``, where ``frames`` is a
CPU ``np.ndarray`` unless TorchCodec decodes on ``device="cuda"``.
"""
target = VideoTargetMetadata(
num_frames=num_frames, fps=fps, max_duration=max_duration
@@ -915,8 +916,7 @@ class Molmo2VideoBackend(VideoLoader):
sampling_fps: float,
max_fps: float = 8.0,
) -> list[float]:
"""
Return the subset of `video_fps` factors that remain multiples
"""Return the subset of `video_fps` factors that remain multiples
of `sampling_fps`.
Examples:
@@ -931,6 +931,7 @@ class Molmo2VideoBackend(VideoLoader):
...
ValueError: sampling_fps=2 must divide video_fps=5 to produce
consistent frame steps.
"""
if sampling_fps is None:
raise ValueError("sampling_fps must be provided")
@@ -967,8 +968,8 @@ class Molmo2VideoBackend(VideoLoader):
frame_sample_mode: str,
candidate_target_fps: list[float],
) -> float | None:
"""
Get the target fps that best spans the videoand has the most frames sampled
"""Get the target fps that best spans the video and samples the most
frames.
"""
num_frames_sampled = 0
selected_target_fps = None
@@ -1272,8 +1273,7 @@ class OpenCVDynamicOpenPanguVideoBackend(VideoLoader):
frame_recovery: bool = False,
**kwargs,
) -> tuple[npt.NDArray, dict[str, Any]]:
"""
Load video frames with dynamic sampling based on duration.
"""Load video frames with dynamic sampling based on duration.
Args:
data: Raw video bytes
@@ -1284,6 +1284,7 @@ class OpenCVDynamicOpenPanguVideoBackend(VideoLoader):
Returns:
Tuple of (frames_array, metadata_dict)
"""
# recompute source metadata with adjusted duration to ensure correct
# sampling indices computation
+2 -2
View File
@@ -128,8 +128,7 @@ class OpenCVVideoBackendMixin:
frame_indices: list[int],
total_frames: int,
) -> tuple[npt.NDArray, list[int], dict[int, int]]:
"""
Read frames with dynamic window forward-scan recovery.
"""Read frames with dynamic window forward-scan recovery.
When a target frame fails to load, the next successfully grabbed
frame (before the next target frame) will be used to recover it.
@@ -144,6 +143,7 @@ class OpenCVVideoBackendMixin:
- frames_array: Array of loaded frames
- valid_frame_indices: List of frame indices that were loaded
- recovered_map: Dict mapping recovered_idx -> source_idx
"""
width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))
height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))
+8 -9
View File
@@ -16,8 +16,7 @@ import torch
def compute_retained_tokens_count(
tokens_per_frame: int, num_frames: int, q: float
) -> int:
"""
Compute the number of retained tokens for a given video.
"""Compute the number of retained tokens for a given video.
Method ensures that we retain all the tokens from the first frame
regardless of the pruning rate.
@@ -28,6 +27,7 @@ def compute_retained_tokens_count(
Returns:
The number of retained tokens.
"""
total_tokens = tokens_per_frame * num_frames
evs_num_tokens = int(total_tokens * (1 - q))
@@ -41,8 +41,7 @@ def compute_retention_mask(
spatial_merge_size: int,
q: float,
) -> torch.Tensor:
"""
Computes the retention mask for input video embeddings.
"""Computes the retention mask for input video embeddings.
Args:
video_embeds (`torch.Tensor`): The input video embeddings
@@ -55,6 +54,7 @@ def compute_retention_mask(
Returns:
`torch.Tensor`: The retention mask for the video embeddings of
`(T * H * W // spatial_merge_size ^ 2)` shape.
"""
T, H, W = map(int, video_size_thw)
@@ -98,8 +98,7 @@ def compute_mrope_for_media(
tokens_per_second: float = 1.0,
video_second_per_grid: float = 1.0,
) -> torch.Tensor:
"""
Computes the mrope for video embeddings based on the grid dimensions.
"""Computes the mrope for video embeddings based on the grid dimensions.
Computed mrope positions match original qwen 2.5 implementation,
but positions are built for media being the first element in sequence.
@@ -113,6 +112,7 @@ def compute_mrope_for_media(
Tensor of shape `(T * H * W, 4)` where last dimension
represents mrope positions [0:3), while the last channel
contains value of llm_grid_w repeated for all positions.
"""
llm_grid_t = video_size_thw[0]
llm_grid_h = video_size_thw[1] // spatial_merge_size
@@ -160,8 +160,7 @@ def recompute_mrope_positions(
image_token_id: int,
video_token_id: int,
) -> tuple[torch.LongTensor, int]:
"""
Update part of input mrope positions.
"""Update part of input mrope positions.
Original mrope_positions are computed incorrectly, so once we prune media
tokens we should reflect this in the mrope positions for the LLM.
@@ -204,8 +203,8 @@ def recompute_mrope_positions(
Returns:
Tuple of (mrope_positions, mrope_position_delta).
"""
"""
# Tensors
positions: torch.LongTensor = typing.cast(
torch.LongTensor, mrope_positions.clone()
+1
View File
@@ -48,6 +48,7 @@ def compute_retention_mask(
Flat bool tensor of shape `(T*H*W/merge^2,)`, True for retained
tokens. The True count equals `compute_retained_tokens_count` so
placeholders sized at prompt-processing time match exactly.
"""
T, H, W = map(int, video_size_thw)
rows = H // spatial_merge_size