Add pydocstyle to the ruff rules (#52136)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Signed-off-by: Harry Mellor <hej.mellor@gmail.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5
parent
804e5377aa
commit
8c1557a79c
@@ -45,6 +45,7 @@ def get_audio_duration(*, y: npt.NDArray[np.floating], sr: float = 22050) -> flo
|
||||
|
||||
Returns:
|
||||
Duration of the audio in seconds.
|
||||
|
||||
"""
|
||||
n_samples = y.shape[-1]
|
||||
return float(n_samples) / sr
|
||||
@@ -71,6 +72,7 @@ class AudioSpec:
|
||||
(no normalization). 1 = mono, 2 = stereo, etc.
|
||||
channel_reduction: Method to reduce channels when input has more
|
||||
channels than target. Only used when reducing channels.
|
||||
|
||||
"""
|
||||
|
||||
target_channels: int | None = 1
|
||||
@@ -119,6 +121,7 @@ def normalize_audio(
|
||||
Raises:
|
||||
ValueError: If audio has unsupported dimensions or channel expansion
|
||||
is requested (e.g., mono to stereo).
|
||||
|
||||
"""
|
||||
if not spec.needs_normalization:
|
||||
return audio
|
||||
@@ -195,6 +198,7 @@ def resample_audio_pyav(
|
||||
|
||||
Returns:
|
||||
Resampled audio with the same shape as the input (1D → 1D, 2D → 2D).
|
||||
|
||||
"""
|
||||
orig_sr_int = int(round(orig_sr))
|
||||
target_sr_int = int(round(target_sr))
|
||||
@@ -312,6 +316,7 @@ def resample_audio_torchaudio(
|
||||
|
||||
Returns:
|
||||
Resampled audio with the same shape as the input (1D → 1D, 2D → 2D).
|
||||
|
||||
"""
|
||||
orig_sr_int = int(round(orig_sr))
|
||||
target_sr_int = int(round(target_sr))
|
||||
@@ -431,6 +436,7 @@ def split_audio(
|
||||
... )
|
||||
>>> len(chunks)
|
||||
3
|
||||
|
||||
"""
|
||||
if audio_data.ndim > 1:
|
||||
raise ValueError(
|
||||
@@ -500,6 +506,7 @@ def find_split_point(
|
||||
... )
|
||||
>>> 16000 <= split_idx <= 17600
|
||||
True
|
||||
|
||||
"""
|
||||
segment = wav[start_idx:end_idx]
|
||||
|
||||
|
||||
+32
-36
@@ -63,12 +63,12 @@ class MultiModalCacheMissError(RuntimeError):
|
||||
|
||||
|
||||
class MultiModalProcessorCacheItem:
|
||||
"""
|
||||
The data to store inside `MultiModalProcessorOnlyCache`.
|
||||
"""The data to store inside `MultiModalProcessorOnlyCache`.
|
||||
|
||||
Args:
|
||||
item: The processed tensor data corresponding to a multi-modal item.
|
||||
prompt_updates: The prompt updates corresponding to `item`.
|
||||
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
@@ -83,8 +83,7 @@ class MultiModalProcessorCacheItem:
|
||||
|
||||
|
||||
class MultiModalProcessorCacheItemMetadata:
|
||||
"""
|
||||
The metadata to store inside `MultiModalProcessorSenderCache`.
|
||||
"""The metadata to store inside `MultiModalProcessorSenderCache`.
|
||||
|
||||
Args:
|
||||
item: The processed tensor data corresponding to a multi-modal item.
|
||||
@@ -94,6 +93,7 @@ class MultiModalProcessorCacheItemMetadata:
|
||||
prompt_updates: The prompt updates corresponding to `item`.
|
||||
This needs to stay on P0 because for some models, they are
|
||||
dependent on the processed tensor data (cached on P1).
|
||||
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
@@ -163,8 +163,7 @@ class MultiModalCache:
|
||||
|
||||
@classmethod
|
||||
def get_item_complexity(cls, value: MultiModalCacheValue) -> int:
|
||||
"""
|
||||
Get the number of leaf elements in a multi-modal cache value.
|
||||
"""Get the number of leaf elements in a multi-modal cache value.
|
||||
|
||||
This provides a measure of structural complexity that can be useful
|
||||
for debugging cache performance and understanding data patterns.
|
||||
@@ -174,6 +173,7 @@ class MultiModalCache:
|
||||
|
||||
Returns:
|
||||
The number of leaf elements in the nested structure.
|
||||
|
||||
"""
|
||||
return json_count_leaves(value)
|
||||
|
||||
@@ -196,8 +196,7 @@ _O = TypeVar("_O", covariant=True)
|
||||
|
||||
|
||||
class BaseMultiModalCache(ABC, Generic[_I, _O]):
|
||||
"""
|
||||
Abstract base class to read/write multi-modal items from cache.
|
||||
"""Abstract base class to read/write multi-modal items from cache.
|
||||
|
||||
The idea of multi-modal caching is based on having a client and server
|
||||
where the client executes in the frontend process (=P0) and
|
||||
@@ -226,8 +225,7 @@ class BaseMultiModalCache(ABC, Generic[_I, _O]):
|
||||
mm_item: _I,
|
||||
mm_hash: str,
|
||||
) -> _O:
|
||||
"""
|
||||
Possibly update a multi-modal item based on whether it is
|
||||
"""Possibly update a multi-modal item based on whether it is
|
||||
in the underlying cache.
|
||||
|
||||
This update is done out-of-place and updates the cache eviction order.
|
||||
@@ -238,6 +236,7 @@ class BaseMultiModalCache(ABC, Generic[_I, _O]):
|
||||
|
||||
Returns:
|
||||
The update multi-modal item.
|
||||
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
@@ -246,8 +245,7 @@ class BaseMultiModalCache(ABC, Generic[_I, _O]):
|
||||
mm_items: Sequence[_I],
|
||||
mm_hashes: list[str],
|
||||
) -> list[_O]:
|
||||
"""
|
||||
Possibly update a sequence of multi-modal items based on whether they
|
||||
"""Possibly update a sequence of multi-modal items based on whether they
|
||||
are in the underlying cache.
|
||||
|
||||
This update is done out-of-place and updates the cache eviction order.
|
||||
@@ -258,6 +256,7 @@ class BaseMultiModalCache(ABC, Generic[_I, _O]):
|
||||
|
||||
Returns:
|
||||
A new list of updated multi-modal items.
|
||||
|
||||
"""
|
||||
assert len(mm_items) == len(mm_hashes)
|
||||
|
||||
@@ -289,6 +288,7 @@ class BaseMultiModalCache(ABC, Generic[_I, _O]):
|
||||
|
||||
Returns:
|
||||
`True` if the item was cached, otherwise `False`.
|
||||
|
||||
"""
|
||||
if cache.put_if_fits(key, value):
|
||||
return True
|
||||
@@ -325,8 +325,7 @@ class BaseMultiModalProcessorCache(
|
||||
|
||||
@abstractmethod
|
||||
def is_cached_item(self, mm_hash: str) -> bool:
|
||||
"""
|
||||
Check whether a multi-modal item is
|
||||
"""Check whether a multi-modal item is
|
||||
in the underlying cache.
|
||||
|
||||
This **DOES NOT** update the cache eviction order.
|
||||
@@ -336,12 +335,12 @@ class BaseMultiModalProcessorCache(
|
||||
|
||||
Returns:
|
||||
`True` if the item is cached, otherwise `False`.
|
||||
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
def is_cached(self, mm_hashes: list[str]) -> list[bool]:
|
||||
"""
|
||||
Check whether a sequence of multi-modal items are
|
||||
"""Check whether a sequence of multi-modal items are
|
||||
in the underlying cache.
|
||||
|
||||
This **DOES NOT** update the cache eviction order.
|
||||
@@ -351,6 +350,7 @@ class BaseMultiModalProcessorCache(
|
||||
|
||||
Returns:
|
||||
For each item, `True` if the item is cached, otherwise `False`.
|
||||
|
||||
"""
|
||||
return [self.is_cached_item(mm_hash) for mm_hash in mm_hashes]
|
||||
|
||||
@@ -367,31 +367,30 @@ class BaseMultiModalProcessorCache(
|
||||
|
||||
@abstractmethod
|
||||
def touch_sender_cache_item(self, mm_hash: str) -> None:
|
||||
"""
|
||||
Update the cache eviction order for a multi-modal item.
|
||||
"""Update the cache eviction order for a multi-modal item.
|
||||
|
||||
This is used to touch the item in the cache without changing
|
||||
its value.
|
||||
|
||||
Args:
|
||||
mm_hash: The hash of the multi-modal item.
|
||||
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
def make_stats(self, *, delta: bool = False) -> CacheInfo:
|
||||
"""
|
||||
Get (and reset) the multi-modal cache stats.
|
||||
"""Get (and reset) the multi-modal cache stats.
|
||||
|
||||
Returns:
|
||||
The current multi-modal caching stats.
|
||||
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
class MultiModalProcessorOnlyCache(BaseMultiModalProcessorCache):
|
||||
"""
|
||||
The cache which is used on P0 when IPC caching is disabled.
|
||||
"""The cache which is used on P0 when IPC caching is disabled.
|
||||
|
||||
How to update each item:
|
||||
|
||||
@@ -442,8 +441,7 @@ class MultiModalProcessorOnlyCache(BaseMultiModalProcessorCache):
|
||||
|
||||
|
||||
class MultiModalProcessorSenderCache(BaseMultiModalProcessorCache):
|
||||
"""
|
||||
The cache which is used on P0 when IPC caching is enabled.
|
||||
"""The cache which is used on P0 when IPC caching is enabled.
|
||||
|
||||
How to update each item:
|
||||
|
||||
@@ -507,8 +505,7 @@ class MultiModalProcessorSenderCache(BaseMultiModalProcessorCache):
|
||||
|
||||
|
||||
class ShmObjectStoreSenderCache(BaseMultiModalProcessorCache):
|
||||
"""
|
||||
The cache which is used on P0 when IPC caching is enabled.
|
||||
"""The cache which is used on P0 when IPC caching is enabled.
|
||||
|
||||
How to update each item:
|
||||
|
||||
@@ -609,7 +606,8 @@ class ShmObjectStoreSenderCache(BaseMultiModalProcessorCache):
|
||||
@override
|
||||
def touch_sender_cache_item(self, mm_hash: str) -> None:
|
||||
"""Touch the item in shared memory cache to prevent eviction.
|
||||
Increments writer_flag on sender side."""
|
||||
Increments writer_flag on sender side.
|
||||
"""
|
||||
self._shm_cache.touch(mm_hash)
|
||||
|
||||
@override
|
||||
@@ -662,8 +660,7 @@ class BaseMultiModalReceiverCache(
|
||||
self,
|
||||
mm_features: list["MultiModalFeatureSpec"],
|
||||
) -> list["MultiModalFeatureSpec"]:
|
||||
"""
|
||||
Update multimodal features with cached encoder outputs.
|
||||
"""Update multimodal features with cached encoder outputs.
|
||||
Touch all identifier at first before update to avoid
|
||||
item in updated list evict during update.
|
||||
|
||||
@@ -695,8 +692,7 @@ class BaseMultiModalReceiverCache(
|
||||
mm_hash: str,
|
||||
mm_item: MultiModalKwargsItem | None = None,
|
||||
) -> None:
|
||||
"""
|
||||
Update the cache eviction order for a multi-modal item.
|
||||
"""Update the cache eviction order for a multi-modal item.
|
||||
|
||||
This is used to touch the item in the cache without changing
|
||||
its value.
|
||||
@@ -705,13 +701,13 @@ class BaseMultiModalReceiverCache(
|
||||
mm_hash: The hash of the multi-modal item.
|
||||
mm_item: The multi-modal item itself. This is optional and
|
||||
may not be needed by some cache implementations.
|
||||
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
class MultiModalReceiverCache(BaseMultiModalReceiverCache):
|
||||
"""
|
||||
The cache which is used on P1 when IPC caching is enabled.
|
||||
"""The cache which is used on P1 when IPC caching is enabled.
|
||||
|
||||
How to update each item:
|
||||
|
||||
@@ -762,8 +758,7 @@ class MultiModalReceiverCache(BaseMultiModalReceiverCache):
|
||||
|
||||
|
||||
class ShmObjectStoreReceiverCache(BaseMultiModalReceiverCache):
|
||||
"""
|
||||
The cache which is used on P1 Worker Process when IPC caching is enabled.
|
||||
"""The cache which is used on P1 Worker Process when IPC caching is enabled.
|
||||
|
||||
How to update each item:
|
||||
|
||||
@@ -827,7 +822,8 @@ class ShmObjectStoreReceiverCache(BaseMultiModalReceiverCache):
|
||||
mm_item: MultiModalKwargsItem | None = None,
|
||||
) -> None:
|
||||
"""Touch the item in shared memory cache to prevent eviction.
|
||||
Increments reader_count on receiver side."""
|
||||
Increments reader_count on receiver side.
|
||||
"""
|
||||
assert mm_item is not None
|
||||
if "address" in mm_item:
|
||||
address = cast(int, mm_item["address"].data)
|
||||
|
||||
@@ -19,8 +19,7 @@ def get_mm_max_toks_per_item(
|
||||
processor: BaseMultiModalProcessor,
|
||||
mm_counts: Mapping[str, int],
|
||||
) -> Mapping[str, int]:
|
||||
"""
|
||||
Get the maximum number of tokens per data item from each modality based
|
||||
"""Get the maximum number of tokens per data item from each modality based
|
||||
on underlying model configuration.
|
||||
"""
|
||||
max_tokens_per_item = processor.info.get_mm_max_tokens_per_item(
|
||||
|
||||
@@ -183,6 +183,7 @@ def reserve_mm_ipc_gpu_memory(
|
||||
|
||||
Raises:
|
||||
ValueError: If the reservation leaves no memory for the KV cache.
|
||||
|
||||
"""
|
||||
if mm_config is None:
|
||||
return available_kv_cache_memory_bytes
|
||||
|
||||
@@ -23,8 +23,7 @@ logger = init_logger(__name__)
|
||||
def _get_hasher_factory(
|
||||
algorithm: MMHasherAlgorithm,
|
||||
) -> Callable[[], "hashlib._Hash"]:
|
||||
"""
|
||||
Get the hasher factory based on the configured algorithm.
|
||||
"""Get the hasher factory based on the configured algorithm.
|
||||
|
||||
Args:
|
||||
algorithm: Hash algorithm name (blake3, sha256, or sha512)
|
||||
@@ -33,8 +32,8 @@ def _get_hasher_factory(
|
||||
Supports blake3 (default), sha256, and sha512 for FIPS compliance.
|
||||
|
||||
See: https://github.com/vllm-project/vllm/issues/18334
|
||||
"""
|
||||
|
||||
"""
|
||||
if algorithm == "blake3":
|
||||
from blake3 import blake3
|
||||
|
||||
|
||||
@@ -38,7 +38,8 @@ def rgba_to_rgb(
|
||||
|
||||
def _has_transparency(image: Image.Image) -> bool:
|
||||
"""Detect whether an image carries transparency data (RGBA, LA, PA,
|
||||
or tRNS chunk in P/L/RGB PNGs)."""
|
||||
or tRNS chunk in P/L/RGB PNGs).
|
||||
"""
|
||||
if image.mode in ("RGBA", "LA", "PA"):
|
||||
return True
|
||||
return "transparency" in getattr(image, "info", {})
|
||||
|
||||
+30
-47
@@ -120,11 +120,9 @@ VisionChunk: TypeAlias = VisionChunkImage | VisionChunkVideo
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class PlaceholderRange:
|
||||
"""
|
||||
Placeholder location information for multi-modal data.
|
||||
"""Placeholder location information for multi-modal data.
|
||||
|
||||
Example:
|
||||
|
||||
Prompt: `AAAA BBBB What is in these images?`
|
||||
|
||||
Images A and B will have:
|
||||
@@ -133,6 +131,7 @@ class PlaceholderRange:
|
||||
A: PlaceholderRange(offset=0, length=4)
|
||||
B: PlaceholderRange(offset=5, length=4)
|
||||
```
|
||||
|
||||
"""
|
||||
|
||||
offset: int
|
||||
@@ -161,8 +160,7 @@ class PlaceholderRange:
|
||||
def get_embeds_indices_in_range(
|
||||
self, start_idx: int, end_idx: int
|
||||
) -> tuple[int, int]:
|
||||
"""
|
||||
Returns the starting and ending indices of the embeddings of encoder outputs
|
||||
"""Returns the starting and ending indices of the embeddings of encoder outputs
|
||||
in the range of [start_idx, end_idx) in the placeholders.
|
||||
|
||||
For example, given:
|
||||
@@ -190,6 +188,7 @@ class PlaceholderRange:
|
||||
A tuple `(start, end)` representing the start and end
|
||||
indices (inclusive) of the embedded region.
|
||||
Returns full placeholder range if `is_embed` is `None`.
|
||||
|
||||
"""
|
||||
if self.is_embed is None:
|
||||
return [(self.offset, self.offset + self.length - 1)]
|
||||
@@ -234,8 +233,7 @@ def nested_tensors_equal(
|
||||
b: NestedTensors,
|
||||
check_dtype: bool = True,
|
||||
) -> bool:
|
||||
"""
|
||||
Equality check between
|
||||
"""Equality check between
|
||||
[`NestedTensors`][vllm.multimodal.inputs.NestedTensors] objects.
|
||||
|
||||
If `check_dtype` is `True`, the tensors must have the same dtype.
|
||||
@@ -297,7 +295,8 @@ def _is_dense(t: torch.Tensor) -> bool:
|
||||
"""Whether `t`'s elements fill a contiguous storage range, possibly
|
||||
permuted (e.g. a transposed matrix). Such layouts can be copied with
|
||||
pitched cudaMemcpy2D/3DAsync directly from pinned memory, whereas
|
||||
gapped layouts (e.g. strided slices) stage through a pageable temp."""
|
||||
gapped layouts (e.g. strided slices) stage through a pageable temp.
|
||||
"""
|
||||
if not t.is_contiguous():
|
||||
expected_stride = 1
|
||||
for stride, size in sorted(zip(t.stride(), t.shape)):
|
||||
@@ -336,8 +335,7 @@ A dictionary containing nested tensors which have been batched via
|
||||
|
||||
|
||||
def batched_tensors_equal(a: BatchedTensorInputs, b: BatchedTensorInputs) -> bool:
|
||||
"""
|
||||
Equality check between
|
||||
"""Equality check between
|
||||
[`BatchedTensorInputs`][vllm.multimodal.inputs.BatchedTensorInputs] objects.
|
||||
"""
|
||||
return all(k in b and nested_tensors_equal(a[k], b[k]) for k in a)
|
||||
@@ -345,8 +343,7 @@ def batched_tensors_equal(a: BatchedTensorInputs, b: BatchedTensorInputs) -> boo
|
||||
|
||||
@dataclass
|
||||
class MultiModalFeatureSpec:
|
||||
"""
|
||||
Represents a single multimodal input with its processed data and metadata.
|
||||
"""Represents a single multimodal input with its processed data and metadata.
|
||||
|
||||
Used to track multimodal data through processing and caching.
|
||||
A request containing multiple multimodal items will have one
|
||||
@@ -392,8 +389,7 @@ class MultiModalFeatureSpec:
|
||||
|
||||
@dataclass
|
||||
class MultiModalFieldElem:
|
||||
"""
|
||||
Represents a processed keyword argument to pass to a model for a
|
||||
"""Represents a processed keyword argument to pass to a model for a
|
||||
[`MultiModalKwargsItem`][vllm.multimodal.inputs.MultiModalKwargsItem].
|
||||
"""
|
||||
|
||||
@@ -429,8 +425,7 @@ class MultiModalFieldElem:
|
||||
|
||||
@dataclass(frozen=True, kw_only=True)
|
||||
class BaseMultiModalField(ABC):
|
||||
"""
|
||||
Defines how to interpret tensor data belonging to a keyword argument for
|
||||
"""Defines how to interpret tensor data belonging to a keyword argument for
|
||||
[`MultiModalKwargsItems`][vllm.multimodal.inputs.MultiModalKwargsItems],
|
||||
and vice versa.
|
||||
"""
|
||||
@@ -458,8 +453,7 @@ class BaseMultiModalField(ABC):
|
||||
key: str,
|
||||
data: NestedTensors,
|
||||
) -> Sequence[MultiModalFieldElem]:
|
||||
"""
|
||||
Construct
|
||||
"""Construct
|
||||
[`MultiModalFieldElem`][vllm.multimodal.inputs.MultiModalFieldElem]
|
||||
instances to represent the provided data.
|
||||
|
||||
@@ -481,8 +475,7 @@ class BaseMultiModalField(ABC):
|
||||
device: torch.types.Device = None,
|
||||
pin_memory: bool = False,
|
||||
) -> NestedTensors:
|
||||
"""
|
||||
Merge the data from multiple instances of
|
||||
"""Merge the data from multiple instances of
|
||||
[`MultiModalFieldElem`][vllm.multimodal.inputs.MultiModalFieldElem].
|
||||
|
||||
This is the inverse of
|
||||
@@ -511,9 +504,8 @@ class BaseMultiModalField(ABC):
|
||||
|
||||
@dataclass(frozen=True, kw_only=True)
|
||||
class MultiModalBatchedField(BaseMultiModalField):
|
||||
"""
|
||||
Info:
|
||||
[`MultiModalFieldConfig.batched`][vllm.multimodal.inputs.MultiModalFieldConfig.batched]
|
||||
"""Info:
|
||||
[`MultiModalFieldConfig.batched`][vllm.multimodal.inputs.MultiModalFieldConfig.batched]
|
||||
"""
|
||||
|
||||
def build_elems(
|
||||
@@ -547,10 +539,9 @@ class MultiModalBatchedField(BaseMultiModalField):
|
||||
|
||||
@dataclass(frozen=True, kw_only=True)
|
||||
class MultiModalFlatField(BaseMultiModalField):
|
||||
"""
|
||||
Info:
|
||||
[`MultiModalFieldConfig.flat`][vllm.multimodal.inputs.MultiModalFieldConfig.flat]
|
||||
[`MultiModalFieldConfig.flat_from_sizes`][vllm.multimodal.inputs.MultiModalFieldConfig.flat_from_sizes]
|
||||
"""Info:
|
||||
[`MultiModalFieldConfig.flat`][vllm.multimodal.inputs.MultiModalFieldConfig.flat]
|
||||
[`MultiModalFieldConfig.flat_from_sizes`][vllm.multimodal.inputs.MultiModalFieldConfig.flat_from_sizes]
|
||||
"""
|
||||
|
||||
slices: Sequence[slice] | Sequence[Sequence[slice]]
|
||||
@@ -637,9 +628,8 @@ class MultiModalFlatField(BaseMultiModalField):
|
||||
|
||||
@dataclass(frozen=True, kw_only=True)
|
||||
class MultiModalSharedField(BaseMultiModalField):
|
||||
"""
|
||||
Info:
|
||||
[`MultiModalFieldConfig.shared`][vllm.multimodal.inputs.MultiModalFieldConfig.shared]
|
||||
"""Info:
|
||||
[`MultiModalFieldConfig.shared`][vllm.multimodal.inputs.MultiModalFieldConfig.shared]
|
||||
"""
|
||||
|
||||
batch_size: int
|
||||
@@ -666,8 +656,7 @@ class MultiModalSharedField(BaseMultiModalField):
|
||||
class MultiModalFieldConfig:
|
||||
@staticmethod
|
||||
def batched(modality: str, *, keep_on_cpu: bool = False):
|
||||
"""
|
||||
Defines a field where an element in the batch is obtained by
|
||||
"""Defines a field where an element in the batch is obtained by
|
||||
indexing into the first dimension of the underlying data.
|
||||
|
||||
Args:
|
||||
@@ -676,7 +665,6 @@ class MultiModalFieldConfig:
|
||||
keep_on_cpu: Whether to keep this field on the CPU for the model inputs.
|
||||
|
||||
Example:
|
||||
|
||||
```
|
||||
Input:
|
||||
Data: [[AAAA]
|
||||
@@ -688,6 +676,7 @@ class MultiModalFieldConfig:
|
||||
Element 2: [BBBB]
|
||||
Element 3: [CCCC]
|
||||
```
|
||||
|
||||
"""
|
||||
return MultiModalFieldConfig(
|
||||
field=MultiModalBatchedField(keep_on_cpu=keep_on_cpu),
|
||||
@@ -702,8 +691,7 @@ class MultiModalFieldConfig:
|
||||
*,
|
||||
keep_on_cpu: bool = False,
|
||||
):
|
||||
"""
|
||||
Defines a field where an element in the batch is obtained by
|
||||
"""Defines a field where an element in the batch is obtained by
|
||||
slicing along the first dimension of the underlying data.
|
||||
|
||||
Args:
|
||||
@@ -716,7 +704,6 @@ class MultiModalFieldConfig:
|
||||
keep_on_cpu: Whether to keep this field on the CPU for the model inputs.
|
||||
|
||||
Example:
|
||||
|
||||
```
|
||||
Given:
|
||||
slices: [slice(0, 3), slice(3, 7), slice(7, 9)]
|
||||
@@ -746,6 +733,7 @@ class MultiModalFieldConfig:
|
||||
Element 2: [[B],[B],[B],[B]]
|
||||
Element 3: [[C],[C]]
|
||||
```
|
||||
|
||||
"""
|
||||
return MultiModalFieldConfig(
|
||||
field=MultiModalFlatField(
|
||||
@@ -764,8 +752,7 @@ class MultiModalFieldConfig:
|
||||
*,
|
||||
keep_on_cpu: bool = False,
|
||||
):
|
||||
"""
|
||||
Defines a field where an element in the batch is obtained by
|
||||
"""Defines a field where an element in the batch is obtained by
|
||||
slicing along the first dimension of the underlying data.
|
||||
|
||||
Args:
|
||||
@@ -777,7 +764,6 @@ class MultiModalFieldConfig:
|
||||
keep_on_cpu: Whether to keep this field on the CPU for the model inputs.
|
||||
|
||||
Example:
|
||||
|
||||
```
|
||||
Given:
|
||||
size_per_item: [3, 4, 2]
|
||||
@@ -807,8 +793,8 @@ class MultiModalFieldConfig:
|
||||
|
||||
Info:
|
||||
[`MultiModalFieldConfig.flat`][vllm.multimodal.inputs.MultiModalFieldConfig.flat]
|
||||
"""
|
||||
|
||||
"""
|
||||
if size_per_item.ndim != 1:
|
||||
raise ValueError(
|
||||
"size_per_item should be a 1-D tensor, "
|
||||
@@ -836,8 +822,7 @@ class MultiModalFieldConfig:
|
||||
*,
|
||||
keep_on_cpu: bool = False,
|
||||
):
|
||||
"""
|
||||
Defines a field where an element in the batch is obtained by
|
||||
"""Defines a field where an element in the batch is obtained by
|
||||
taking the entirety of the underlying data.
|
||||
|
||||
This means that the data is the same for each element in the batch.
|
||||
@@ -849,7 +834,6 @@ class MultiModalFieldConfig:
|
||||
keep_on_cpu: Whether to keep this field on the CPU for the model inputs.
|
||||
|
||||
Example:
|
||||
|
||||
```
|
||||
Given:
|
||||
batch_size: 4
|
||||
@@ -863,6 +847,7 @@ class MultiModalFieldConfig:
|
||||
Element 3: [XYZ]
|
||||
Element 4: [XYZ]
|
||||
```
|
||||
|
||||
"""
|
||||
return MultiModalFieldConfig(
|
||||
field=MultiModalSharedField(
|
||||
@@ -884,8 +869,7 @@ class MultiModalFieldConfig:
|
||||
|
||||
|
||||
class MultiModalKwargsItem(UserDict[str, MultiModalFieldElem]):
|
||||
"""
|
||||
A dictionary of processed keyword arguments to pass to the model,
|
||||
"""A dictionary of processed keyword arguments to pass to the model,
|
||||
corresponding to a single item in
|
||||
[`MultiModalDataItems`][vllm.multimodal.parse.MultiModalDataItems].
|
||||
"""
|
||||
@@ -912,8 +896,7 @@ _I = TypeVar(
|
||||
|
||||
|
||||
class MultiModalKwargsItems(UserDict[str, Sequence[_I]]):
|
||||
"""
|
||||
A dictionary of processed multi-modal inputs by modality.
|
||||
"""A dictionary of processed multi-modal inputs by modality.
|
||||
|
||||
For example, given a processor that processes
|
||||
images into `pixel_values` and `image_grid_thw`,
|
||||
|
||||
@@ -92,6 +92,10 @@ def load_audio_pyav(
|
||||
Args:
|
||||
path: A :class:`~io.BytesIO` buffer, a filesystem
|
||||
:class:`~pathlib.Path`, or a string path.
|
||||
sr: Target sample rate, or None to keep the native rate.
|
||||
mono: Whether to average the channels down to mono.
|
||||
max_decode_bytes: If set, abort decoding once this many bytes have
|
||||
been read from the source.
|
||||
max_duration_s: If set, abort decoding once the accumulated
|
||||
sample count exceeds this many seconds of audio. Prevents
|
||||
decompression-bomb attacks where a small compressed file
|
||||
@@ -100,6 +104,7 @@ def load_audio_pyav(
|
||||
Returns:
|
||||
``(waveform, sample_rate)`` where *waveform* is a 1-D float32
|
||||
NumPy array and *sample_rate* is the native sample rate in Hz.
|
||||
|
||||
"""
|
||||
try:
|
||||
container = av.open(path)
|
||||
@@ -209,7 +214,7 @@ def load_audio_soundfile(
|
||||
max_duration_s: float | None = None,
|
||||
max_decode_bytes: int | None = None,
|
||||
) -> tuple[np.ndarray, int]:
|
||||
"""Load audio via soundfile"""
|
||||
"""Load audio via soundfile."""
|
||||
with soundfile.SoundFile(path) as f:
|
||||
native_sr = f.samplerate
|
||||
if max_duration_s is not None:
|
||||
@@ -274,6 +279,7 @@ def load_audio_torchcodec(
|
||||
``(waveform, sample_rate)`` where *waveform* is a float32 NumPy
|
||||
array (1-D when ``mono=True``) and *sample_rate* is the output
|
||||
sample rate in Hz.
|
||||
|
||||
"""
|
||||
if AudioDecoder is None:
|
||||
# Unify "torchcodec not installed" and "system ffmpeg missing" into
|
||||
@@ -412,10 +418,17 @@ def load_audio(
|
||||
"""Load audio using the selected decoding backend.
|
||||
|
||||
Args:
|
||||
path: Audio file path or in-memory buffer.
|
||||
sr: Target sample rate, or None to keep the native rate.
|
||||
mono: Whether to downmix to a single channel.
|
||||
max_duration_s: Reject audio longer than this many seconds.
|
||||
max_decode_bytes: Reject audio that would decode to more than this
|
||||
many bytes.
|
||||
backend: One of ``AUDIO_BACKENDS``. ``None`` (default) selects
|
||||
``"auto"``, which tries soundfile, then torchcodec, then PyAV;
|
||||
the other values select a single
|
||||
backend with no fallback.
|
||||
|
||||
"""
|
||||
backend = backend or "auto"
|
||||
if backend not in AUDIO_BACKENDS:
|
||||
|
||||
@@ -14,8 +14,7 @@ _T = TypeVar("_T")
|
||||
|
||||
@dataclass
|
||||
class MediaWithBytes(Generic[_T]):
|
||||
"""
|
||||
Wrapper that couples a media object with its original encoded bytes.
|
||||
"""Wrapper that couples a media object with its original encoded bytes.
|
||||
|
||||
This ensures the raw bytes and media object remain synchronized,
|
||||
preventing cache corruption from in-place modifications.
|
||||
@@ -89,8 +88,7 @@ class MediaIO(ABC, Generic[_T]):
|
||||
|
||||
@abstractmethod
|
||||
def load_base64(self, media_type: str, data: str) -> _T:
|
||||
"""
|
||||
List of media types:
|
||||
"""List of media types:
|
||||
https://www.iana.org/assignments/media-types/media-types.xhtml
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
@@ -73,6 +73,7 @@ def _wrap_media_fetch_error(
|
||||
408/429, invalid URL)
|
||||
Original exception for transient errors (5xx, 408, 429, network blips)
|
||||
or other exceptions
|
||||
|
||||
"""
|
||||
if isinstance(exc, VLLMValidationError):
|
||||
return exc
|
||||
@@ -162,16 +163,16 @@ class MediaConnector:
|
||||
allowed_local_media_path: str = "",
|
||||
allowed_media_domains: list[str] | None = None,
|
||||
) -> None:
|
||||
"""
|
||||
Args:
|
||||
media_io_kwargs: Additional args passed to process media
|
||||
inputs, keyed by modalities. For example,
|
||||
to set num_frames for video, set
|
||||
`--media-io-kwargs '{"video":{"num_frames":40}}'`
|
||||
connection: HTTP connection client to download media contents.
|
||||
allowed_local_media_path: A local directory to load media files from.
|
||||
allowed_media_domains: If set, only media URLs that belong to this
|
||||
domain can be used for multi-modal inputs.
|
||||
"""Args:
|
||||
media_io_kwargs: Additional args passed to process media
|
||||
inputs, keyed by modalities. For example,
|
||||
to set num_frames for video, set
|
||||
`--media-io-kwargs '{"video":{"num_frames":40}}'`
|
||||
connection: HTTP connection client to download media contents.
|
||||
allowed_local_media_path: A local directory to load media files from.
|
||||
allowed_media_domains: If set, only media URLs that belong to this
|
||||
domain can be used for multi-modal inputs.
|
||||
|
||||
"""
|
||||
super().__init__()
|
||||
|
||||
@@ -468,9 +469,7 @@ class MediaConnector:
|
||||
self,
|
||||
audio_url: str,
|
||||
) -> tuple[np.ndarray, int | float]:
|
||||
"""
|
||||
Load audio from a URL.
|
||||
"""
|
||||
"""Load audio from a URL."""
|
||||
audio_io = AudioMediaIO(**self.media_io_kwargs.get("audio", {}))
|
||||
|
||||
return self.load_from_url(
|
||||
@@ -483,9 +482,7 @@ class MediaConnector:
|
||||
self,
|
||||
audio_url: str,
|
||||
) -> tuple[np.ndarray, int | float]:
|
||||
"""
|
||||
Asynchronously fetch audio from a URL.
|
||||
"""
|
||||
"""Asynchronously fetch audio from a URL."""
|
||||
audio_io = AudioMediaIO(**self.media_io_kwargs.get("audio", {}))
|
||||
|
||||
return await self.load_from_url_async(
|
||||
@@ -500,8 +497,7 @@ class MediaConnector:
|
||||
*,
|
||||
image_mode: str | None = "RGB",
|
||||
) -> Image.Image:
|
||||
"""
|
||||
Load a PIL image from an HTTP or base64 data URL.
|
||||
"""Load a PIL image from an HTTP or base64 data URL.
|
||||
|
||||
By default, the image is converted into RGB format. Set
|
||||
`media_io_kwargs={"image": {"image_mode": None}}` to keep the
|
||||
@@ -527,8 +523,7 @@ class MediaConnector:
|
||||
*,
|
||||
image_mode: str | None = "RGB",
|
||||
) -> Image.Image:
|
||||
"""
|
||||
Asynchronously load a PIL image from an HTTP or base64 data URL.
|
||||
"""Asynchronously load a PIL image from an HTTP or base64 data URL.
|
||||
|
||||
By default, the image is converted into RGB format. Set
|
||||
`media_io_kwargs={"image": {"image_mode": None}}` to keep the
|
||||
@@ -555,9 +550,7 @@ class MediaConnector:
|
||||
image_mode: str | None = "RGB",
|
||||
video_processor: str | None = None,
|
||||
) -> MediaWithBytes[tuple[npt.NDArray, dict[str, Any]]]:
|
||||
"""
|
||||
Load video from an HTTP or base64 data URL.
|
||||
"""
|
||||
"""Load video from an HTTP or base64 data URL."""
|
||||
image_io = ImageMediaIO(
|
||||
**({"image_mode": image_mode} | self.media_io_kwargs.get("image", {}))
|
||||
)
|
||||
@@ -581,8 +574,7 @@ class MediaConnector:
|
||||
image_mode: str | None = "RGB",
|
||||
video_processor: str | None = None,
|
||||
) -> MediaWithBytes[tuple[npt.NDArray, dict[str, Any]]]:
|
||||
"""
|
||||
Asynchronously load video from an HTTP or base64 data URL.
|
||||
"""Asynchronously load video from an HTTP or base64 data URL.
|
||||
|
||||
By default, the image is converted into RGB format. Set
|
||||
`media_io_kwargs={"image": {"image_mode": None}}` to keep the
|
||||
@@ -608,9 +600,7 @@ class MediaConnector:
|
||||
self,
|
||||
data: str,
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Load image embedding from a URL.
|
||||
"""
|
||||
"""Load image embedding from a URL."""
|
||||
image_embedding_io = ImageEmbeddingMediaIO()
|
||||
|
||||
return image_embedding_io.load_base64("", data)
|
||||
@@ -619,9 +609,7 @@ class MediaConnector:
|
||||
self,
|
||||
data: str,
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Asynchronously load image embedding from a URL.
|
||||
"""
|
||||
"""Asynchronously load image embedding from a URL."""
|
||||
image_embedding_io = ImageEmbeddingMediaIO()
|
||||
loop = asyncio.get_running_loop()
|
||||
|
||||
@@ -633,9 +621,7 @@ class MediaConnector:
|
||||
self,
|
||||
data: str,
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Load audio embedding from a URL.
|
||||
"""
|
||||
"""Load audio embedding from a URL."""
|
||||
audio_embedding_io = AudioEmbeddingMediaIO()
|
||||
|
||||
return audio_embedding_io.load_base64("", data)
|
||||
@@ -644,9 +630,7 @@ class MediaConnector:
|
||||
self,
|
||||
data: str,
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Asynchronously load audio embedding from a URL.
|
||||
"""
|
||||
"""Asynchronously load audio embedding from a URL."""
|
||||
audio_embedding_io = AudioEmbeddingMediaIO()
|
||||
loop = asyncio.get_running_loop()
|
||||
|
||||
@@ -658,9 +642,7 @@ class MediaConnector:
|
||||
self,
|
||||
data: str,
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Load video embedding from a URL.
|
||||
"""
|
||||
"""Load video embedding from a URL."""
|
||||
video_embedding_io = VideoEmbeddingMediaIO()
|
||||
|
||||
return video_embedding_io.load_base64("", data)
|
||||
@@ -669,9 +651,7 @@ class MediaConnector:
|
||||
self,
|
||||
data: str,
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Asynchronously load video embedding from a URL.
|
||||
"""
|
||||
"""Asynchronously load video embedding from a URL."""
|
||||
video_embedding_io = VideoEmbeddingMediaIO()
|
||||
loop = asyncio.get_running_loop()
|
||||
|
||||
|
||||
+22
-28
@@ -58,8 +58,7 @@ else:
|
||||
|
||||
|
||||
class ModalityDataItems(ABC, Generic[_T, _I]):
|
||||
"""
|
||||
Represents data items for a modality in
|
||||
"""Represents data items for a modality in
|
||||
[`MultiModalDataItems`][vllm.multimodal.parse.MultiModalDataItems].
|
||||
"""
|
||||
|
||||
@@ -151,6 +150,7 @@ def validate_embedding_ndim(
|
||||
tensor: The tensor to validate.
|
||||
modality: The modality name for error messages (e.g., "image", "audio").
|
||||
index: Optional index for list items, included in error messages.
|
||||
|
||||
"""
|
||||
if tensor.ndim < 2 or tensor.ndim > 3:
|
||||
idx_str = f" [{index}]" if index is not None else ""
|
||||
@@ -164,8 +164,7 @@ def validate_embedding_ndim(
|
||||
class EmbeddingItems(
|
||||
ModalityDataItems[torch.Tensor | list[torch.Tensor], torch.Tensor]
|
||||
):
|
||||
"""
|
||||
Base class for data items that are expressed as a batched embedding tensor,
|
||||
"""Base class for data items that are expressed as a batched embedding tensor,
|
||||
or a list of embedding tensors (one per item).
|
||||
"""
|
||||
|
||||
@@ -251,8 +250,7 @@ class EmbeddingItems(
|
||||
class DictEmbeddingItems(
|
||||
ModalityDataItems[Mapping[str, torch.Tensor], Mapping[str, torch.Tensor]]
|
||||
):
|
||||
"""
|
||||
Base class for data items that are expressed as a dictionary of tensors.
|
||||
"""Base class for data items that are expressed as a dictionary of tensors.
|
||||
|
||||
Usually, the dictionary keys correspond to the outputs of HF processor.
|
||||
"""
|
||||
@@ -268,18 +266,18 @@ class DictEmbeddingItems(
|
||||
],
|
||||
optional_fields: Set[str] = frozenset(),
|
||||
) -> None:
|
||||
"""
|
||||
Args:
|
||||
data: The dictionary of tensors for this modality.
|
||||
modality: The modality these items belong to.
|
||||
required_fields: Fields `data` must contain.
|
||||
fields_factory: Builds the field config from the data.
|
||||
optional_fields: Fields `data` may omit. Which fields these are is
|
||||
the caller's decision -- see
|
||||
`MultiModalDataParser.embedding_field_sets`, where a deployment
|
||||
that receives embeddings through an EC connector makes the
|
||||
embeddings optional. They still need a field config, since they
|
||||
are used whenever they *are* supplied.
|
||||
"""Args:
|
||||
data: The dictionary of tensors for this modality.
|
||||
modality: The modality these items belong to.
|
||||
required_fields: Fields `data` must contain.
|
||||
fields_factory: Builds the field config from the data.
|
||||
optional_fields: Fields `data` may omit. Which fields these are is
|
||||
the caller's decision -- see
|
||||
`MultiModalDataParser.embedding_field_sets`, where a deployment
|
||||
that receives embeddings through an EC connector makes the
|
||||
embeddings optional. They still need a field config, since they
|
||||
are used whenever they *are* supplied.
|
||||
|
||||
"""
|
||||
from transformers.feature_extraction_utils import BatchFeature
|
||||
|
||||
@@ -471,14 +469,12 @@ _D = TypeVar("_D", bound=ModalityDataItems[Any, Any])
|
||||
|
||||
|
||||
class MultiModalDataItems(UserDict[str, ModalityDataItems[Any, Any]]):
|
||||
"""
|
||||
A normalized [`MultiModalDataDict`][vllm.inputs.MultiModalDataDict]
|
||||
"""A normalized [`MultiModalDataDict`][vllm.inputs.MultiModalDataDict]
|
||||
such that each entry corresponds to a list.
|
||||
"""
|
||||
|
||||
def select(self, modalities: Set[str]):
|
||||
"""
|
||||
Construct a new `MultiModalDataItems` instance containing only the
|
||||
"""Construct a new `MultiModalDataItems` instance containing only the
|
||||
selected modalities.
|
||||
"""
|
||||
return MultiModalDataItems(
|
||||
@@ -486,8 +482,7 @@ class MultiModalDataItems(UserDict[str, ModalityDataItems[Any, Any]]):
|
||||
)
|
||||
|
||||
def get_count(self, modality: str, *, strict: bool = True) -> int:
|
||||
"""
|
||||
Get the number of data items belonging to a modality.
|
||||
"""Get the number of data items belonging to a modality.
|
||||
|
||||
If `strict=False`, return `0` instead of raising [`KeyError`][]
|
||||
even if the modality is not found.
|
||||
@@ -513,8 +508,7 @@ class MultiModalDataItems(UserDict[str, ModalityDataItems[Any, Any]]):
|
||||
modality: str,
|
||||
typ: type[_D] | tuple[type[_D], ...],
|
||||
) -> _D:
|
||||
"""
|
||||
Get the data items belonging to a modality,
|
||||
"""Get the data items belonging to a modality,
|
||||
requiring that they belong to a certain type.
|
||||
"""
|
||||
if modality not in self:
|
||||
@@ -541,8 +535,7 @@ ModalityDataParser: TypeAlias = Callable[
|
||||
|
||||
|
||||
class MultiModalDataParser:
|
||||
"""
|
||||
Parses [`MultiModalDataDict`][vllm.inputs.MultiModalDataDict]
|
||||
"""Parses [`MultiModalDataDict`][vllm.inputs.MultiModalDataDict]
|
||||
into [`MultiModalDataItems`][vllm.multimodal.parse.MultiModalDataItems].
|
||||
|
||||
Args:
|
||||
@@ -561,6 +554,7 @@ class MultiModalDataParser:
|
||||
allow_missing_mm_embeddings (bool): Whether pre-computed embedding
|
||||
tensors may be absent from the request on a disaggregated consumer.
|
||||
Derived by `BaseProcessingInfo.allow_missing_mm_embeddings`.
|
||||
|
||||
"""
|
||||
|
||||
embedding_fields: Mapping[str, Mapping[str, EmbeddingFieldRole]] = {}
|
||||
|
||||
@@ -70,6 +70,7 @@ def overlay_modality_mm_kwargs(
|
||||
|
||||
Returns:
|
||||
A new dict with the modality-scoped keys overlaid when present.
|
||||
|
||||
"""
|
||||
merged = dict(kwargs)
|
||||
if modality is None:
|
||||
@@ -128,8 +129,7 @@ _P = TypeVar("_P", bound=ProcessorMixin, default=ProcessorMixin)
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class InputProcessingContext:
|
||||
"""
|
||||
Contains information about the model which may be used to
|
||||
"""Contains information about the model which may be used to
|
||||
modify the inputs.
|
||||
"""
|
||||
|
||||
@@ -162,13 +162,13 @@ class InputProcessingContext:
|
||||
typ: type[Any] | tuple[type[Any], ...] | None = None,
|
||||
/,
|
||||
) -> Any:
|
||||
"""
|
||||
Get the HuggingFace configuration
|
||||
"""Get the HuggingFace configuration
|
||||
(`transformers.PretrainedConfig`) of the model,
|
||||
additionally checking its type.
|
||||
|
||||
Raises:
|
||||
TypeError: If the configuration is not of the specified type.
|
||||
|
||||
"""
|
||||
if typ is None:
|
||||
from transformers.configuration_utils import PretrainedConfig
|
||||
@@ -186,17 +186,15 @@ class InputProcessingContext:
|
||||
return hf_config
|
||||
|
||||
def get_hf_image_processor_config(self) -> dict[str, Any]:
|
||||
"""
|
||||
Get the HuggingFace image processor configuration of the model.
|
||||
"""
|
||||
"""Get the HuggingFace image processor configuration of the model."""
|
||||
return self.model_config.hf_image_processor_config
|
||||
|
||||
def get_mm_config(self):
|
||||
"""
|
||||
Get the multimodal config of the model.
|
||||
"""Get the multimodal config of the model.
|
||||
|
||||
Raises:
|
||||
RuntimeError: If the model is not a multimodal model.
|
||||
|
||||
"""
|
||||
mm_config = self.model_config.multimodal_config
|
||||
if mm_config is None:
|
||||
@@ -221,13 +219,13 @@ class InputProcessingContext:
|
||||
/,
|
||||
**kwargs: object,
|
||||
) -> Any:
|
||||
"""
|
||||
Get the HuggingFace processor
|
||||
"""Get the HuggingFace processor
|
||||
(`transformers.ProcessorMixin`) of the model,
|
||||
additionally checking its type.
|
||||
|
||||
Raises:
|
||||
TypeError: If the processor is not of the specified type.
|
||||
|
||||
"""
|
||||
if typ is None:
|
||||
from transformers.processing_utils import ProcessorMixin
|
||||
@@ -254,8 +252,7 @@ class InputProcessingContext:
|
||||
/,
|
||||
**kwargs: object,
|
||||
) -> _T:
|
||||
"""
|
||||
Initialize a HuggingFace-like processor class, merging the
|
||||
"""Initialize a HuggingFace-like processor class, merging the
|
||||
keyword arguments with those in the model's configuration.
|
||||
"""
|
||||
merged_kwargs = self.get_merged_mm_kwargs(kwargs)
|
||||
@@ -317,8 +314,7 @@ class InputProcessingContext:
|
||||
data: Mapping[str, object],
|
||||
kwargs: Mapping[str, object] = {},
|
||||
) -> BatchFeature:
|
||||
"""
|
||||
Call `hf_processor` on the prompt `data`
|
||||
"""Call `hf_processor` on the prompt `data`
|
||||
(text, image, audio...) with configurable options `kwargs`.
|
||||
"""
|
||||
assert callable(hf_processor)
|
||||
@@ -385,8 +381,7 @@ class BaseProcessingInfo:
|
||||
return self.ctx.get_hf_config()
|
||||
|
||||
def get_hf_processor(self, **kwargs: object) -> ProcessorMixin:
|
||||
"""
|
||||
Subclasses can override this method to handle
|
||||
"""Subclasses can override this method to handle
|
||||
specific kwargs from model config or user inputs.
|
||||
"""
|
||||
return self.ctx.get_hf_processor(**kwargs)
|
||||
@@ -409,8 +404,7 @@ class BaseProcessingInfo:
|
||||
return self.get_default_tok_params()
|
||||
|
||||
def _get_expected_hidden_size(self) -> int | None:
|
||||
"""
|
||||
Get expected hidden size for embedding validation if `mm_embeds` are enabled.
|
||||
"""Get expected hidden size for embedding validation if `mm_embeds` are enabled.
|
||||
|
||||
This validates hidden dimensions to prevent a vulnerability where embeddings
|
||||
with correct `ndim` but wrong `shape` could cause crashes at inference time.
|
||||
@@ -430,8 +424,7 @@ class BaseProcessingInfo:
|
||||
return mm_config is not None and mm_config.allow_missing_mm_embeddings
|
||||
|
||||
def get_data_parser(self) -> MultiModalDataParser:
|
||||
"""
|
||||
Constructs a parser to preprocess multi-modal data items
|
||||
"""Constructs a parser to preprocess multi-modal data items
|
||||
before passing them to
|
||||
[`_get_hf_mm_data`][vllm.multimodal.processing.BaseMultiModalProcessor._get_hf_mm_data].
|
||||
|
||||
@@ -453,8 +446,7 @@ class BaseProcessingInfo:
|
||||
return False
|
||||
|
||||
def get_supported_mm_limits(self) -> Mapping[str, int | None]:
|
||||
"""
|
||||
Return the maximum supported number of items for each modality.
|
||||
"""Return the maximum supported number of items for each modality.
|
||||
|
||||
A value of `None` means unlimited number of items.
|
||||
|
||||
@@ -486,8 +478,7 @@ class BaseProcessingInfo:
|
||||
return allowed_limits
|
||||
|
||||
def validate_num_items(self, modality: str, num_items: int) -> None:
|
||||
"""
|
||||
Raise `ValueError` if the number of input items for the given modality
|
||||
"""Raise `ValueError` if the number of input items for the given modality
|
||||
is invalid.
|
||||
"""
|
||||
supported_limit = self.supported_mm_limits.get(modality, 0)
|
||||
@@ -512,8 +503,7 @@ class BaseProcessingInfo:
|
||||
*,
|
||||
validate: bool = True,
|
||||
) -> MultiModalDataItems:
|
||||
"""
|
||||
Normalize [`MultiModalDataDict`][vllm.inputs.MultiModalDataDict]
|
||||
"""Normalize [`MultiModalDataDict`][vllm.inputs.MultiModalDataDict]
|
||||
to [`MultiModalDataItems`][vllm.multimodal.parse.MultiModalDataItems]
|
||||
before passing them to
|
||||
[`_get_hf_mm_data`][vllm.multimodal.processing.BaseMultiModalProcessor._get_hf_mm_data].
|
||||
@@ -546,8 +536,7 @@ class BaseProcessingInfo:
|
||||
seq_len: int,
|
||||
mm_counts: Mapping[str, int],
|
||||
) -> Mapping[str, int] | None:
|
||||
"""
|
||||
Return the maximum number of tokens per item of for each modality.
|
||||
"""Return the maximum number of tokens per item of for each modality.
|
||||
|
||||
When `None` (the default) is returned, vLLM will generate dummy inputs
|
||||
(images/videos) at maximum possible sizes and process them to determine
|
||||
@@ -564,5 +553,6 @@ class BaseProcessingInfo:
|
||||
length and the maximum number of items of each modality allowed,
|
||||
and agree with dummy inputs (images/videos) at maximum possible
|
||||
sizes.
|
||||
|
||||
"""
|
||||
return None
|
||||
|
||||
@@ -26,8 +26,7 @@ logger = init_logger(__name__)
|
||||
|
||||
|
||||
class BaseDummyInputsBuilder(ABC, Generic[_I]):
|
||||
"""
|
||||
Abstract base class that constructs the dummy data to profile
|
||||
"""Abstract base class that constructs the dummy data to profile
|
||||
multi-modal models.
|
||||
"""
|
||||
|
||||
@@ -38,9 +37,7 @@ class BaseDummyInputsBuilder(ABC, Generic[_I]):
|
||||
|
||||
@abstractmethod
|
||||
def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||
"""
|
||||
Build the text input corresponding to `mm_counts`.
|
||||
"""
|
||||
"""Build the text input corresponding to `mm_counts`."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
@@ -50,8 +47,7 @@ class BaseDummyInputsBuilder(ABC, Generic[_I]):
|
||||
mm_counts: Mapping[str, int],
|
||||
mm_options: Mapping[str, BaseDummyOptions],
|
||||
) -> MultiModalDataDict:
|
||||
"""
|
||||
Build the multimodal input which, after processing, results in
|
||||
"""Build the multimodal input which, after processing, results in
|
||||
the maximum possible number of placeholder tokens.
|
||||
|
||||
Args:
|
||||
@@ -61,6 +57,7 @@ class BaseDummyInputsBuilder(ABC, Generic[_I]):
|
||||
If None, use model defaults for backward compatibility.
|
||||
If provided, models can use these to customize dummy
|
||||
data generation.
|
||||
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
@@ -70,14 +67,14 @@ class BaseDummyInputsBuilder(ABC, Generic[_I]):
|
||||
mm_counts: Mapping[str, int],
|
||||
mm_options: Mapping[str, BaseDummyOptions],
|
||||
) -> ProcessorInputs:
|
||||
"""
|
||||
Build the input which, after processing, results in
|
||||
"""Build the input which, after processing, results in
|
||||
the maximum possible number of placeholder tokens.
|
||||
|
||||
Args:
|
||||
seq_len: Sequence length
|
||||
mm_counts: Count of items per modality
|
||||
mm_options: Configurable options per modality (optional)
|
||||
|
||||
"""
|
||||
dummy_text = self.get_dummy_text(mm_counts)
|
||||
dummy_mm_data = self.get_dummy_mm_data(seq_len, mm_counts, mm_options)
|
||||
|
||||
@@ -18,8 +18,7 @@ _HF_MODALITY_PROCESSOR_KWARGS = {
|
||||
|
||||
@dataclass
|
||||
class ProcessorInputs:
|
||||
"""
|
||||
Represents the keyword arguments to
|
||||
"""Represents the keyword arguments to
|
||||
[`vllm.multimodal.processing.BaseMultiModalProcessor.apply`][].
|
||||
"""
|
||||
|
||||
|
||||
@@ -104,8 +104,7 @@ class PromptIndex:
|
||||
class PromptIndexTargets:
|
||||
@staticmethod
|
||||
def start() -> PromptIndex:
|
||||
"""
|
||||
Resolves to the start of the prompt (before the first token).
|
||||
"""Resolves to the start of the prompt (before the first token).
|
||||
|
||||
This results in a match even if the prompt is empty.
|
||||
"""
|
||||
@@ -116,9 +115,7 @@ class PromptIndexTargets:
|
||||
|
||||
@staticmethod
|
||||
def prefix(seq: list[int]) -> PromptIndex:
|
||||
"""
|
||||
Resolves to a location in the prompt after the given prefix.
|
||||
"""
|
||||
"""Resolves to a location in the prompt after the given prefix."""
|
||||
|
||||
def get_match_index(
|
||||
prompt: list[int],
|
||||
@@ -146,8 +143,7 @@ class PromptIndexTargets:
|
||||
|
||||
@staticmethod
|
||||
def end() -> PromptIndex:
|
||||
"""
|
||||
Resolves to the end of the prompt (after the last token).
|
||||
"""Resolves to the end of the prompt (after the last token).
|
||||
|
||||
This results in a match even if the prompt is empty.
|
||||
"""
|
||||
@@ -247,8 +243,7 @@ class UpdateMode(str, Enum):
|
||||
|
||||
@dataclass
|
||||
class PromptUpdate(ABC):
|
||||
"""
|
||||
Defines how to update a prompt with placeholder tokens.
|
||||
"""Defines how to update a prompt with placeholder tokens.
|
||||
|
||||
Note:
|
||||
The target and content are token sequences. When converting text
|
||||
@@ -256,6 +251,7 @@ class PromptUpdate(ABC):
|
||||
`add_special_tokens=False` to match how the prompt itself is
|
||||
tokenized. Otherwise, the updated prompt may contain duplicated
|
||||
special tokens or fail to match the target.
|
||||
|
||||
"""
|
||||
|
||||
modality: str
|
||||
@@ -294,8 +290,7 @@ class PromptUpdate(ABC):
|
||||
return content
|
||||
|
||||
def resolve(self, item_idx: int) -> "ResolvedPromptUpdate":
|
||||
"""
|
||||
Given the index of the processed item within
|
||||
"""Given the index of the processed item within
|
||||
[`modality`][vllm.multimodal.processing.PromptUpdate.modality],
|
||||
output a copy of this object with its lazy attributes resolved.
|
||||
"""
|
||||
@@ -310,11 +305,9 @@ class PromptUpdate(ABC):
|
||||
|
||||
@dataclass
|
||||
class PromptInsertion(PromptUpdate):
|
||||
"""
|
||||
Defines how to insert placeholder tokens into a prompt.
|
||||
"""Defines how to insert placeholder tokens into a prompt.
|
||||
|
||||
Example:
|
||||
|
||||
For each image, insert a number of `<image>` feature placeholders
|
||||
equal to the feature size of the vision encoder after the `<s>` token:
|
||||
|
||||
@@ -355,6 +348,7 @@ class PromptInsertion(PromptUpdate):
|
||||
insertion=[image_token_id] * image_feature_size,
|
||||
)
|
||||
```
|
||||
|
||||
"""
|
||||
|
||||
insertion: PromptUpdateContent = field(repr=False)
|
||||
@@ -379,11 +373,9 @@ class PromptInsertion(PromptUpdate):
|
||||
|
||||
@dataclass
|
||||
class PromptReplacement(PromptUpdate):
|
||||
"""
|
||||
Defines how to replace portions of an input prompt with placeholder tokens.
|
||||
"""Defines how to replace portions of an input prompt with placeholder tokens.
|
||||
|
||||
Example:
|
||||
|
||||
For each image, replace one `<image>` input placeholder in the prompt
|
||||
with a number of `<image>` feature placeholders
|
||||
equal to the feature size of the vision encoder:
|
||||
@@ -412,6 +404,7 @@ class PromptReplacement(PromptUpdate):
|
||||
),
|
||||
)
|
||||
```
|
||||
|
||||
"""
|
||||
|
||||
replacement: PromptUpdateContent = field(repr=False)
|
||||
@@ -447,8 +440,7 @@ _M = TypeVar("_M", bound=_HasModalityAttr | _HasModalityProp)
|
||||
|
||||
|
||||
def full_groupby_modality(values: Iterable[_M]) -> ItemsView[str, list[_M]]:
|
||||
"""
|
||||
Convenience function to apply
|
||||
"""Convenience function to apply
|
||||
[`full_groupby`][vllm.utils.collection_utils.full_groupby]
|
||||
based on modality.
|
||||
"""
|
||||
@@ -462,8 +454,7 @@ class PromptTargetMatch(NamedTuple):
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ResolvedPromptUpdate:
|
||||
"""
|
||||
A [`PromptUpdate`][vllm.multimodal.processing.PromptUpdate] with its
|
||||
"""A [`PromptUpdate`][vllm.multimodal.processing.PromptUpdate] with its
|
||||
lazy attributes resolved.
|
||||
"""
|
||||
|
||||
@@ -522,8 +513,7 @@ def iter_token_matches(
|
||||
*,
|
||||
start_idx: int = 0,
|
||||
) -> Generator[_TokenMatch]:
|
||||
"""
|
||||
Yield each occurrence of `match_ids` in `token_ids`.
|
||||
"""Yield each occurrence of `match_ids` in `token_ids`.
|
||||
|
||||
Note that empty matches are ignored.
|
||||
"""
|
||||
@@ -562,8 +552,7 @@ def replace_token_matches(
|
||||
match_ids: list[int],
|
||||
new_ids: list[int],
|
||||
) -> list[int]:
|
||||
"""
|
||||
Replace each occurrence of `match_ids` in `token_ids`
|
||||
"""Replace each occurrence of `match_ids` in `token_ids`
|
||||
with `new_ids`.
|
||||
|
||||
Note that empty matches are ignored.
|
||||
@@ -846,8 +835,7 @@ def apply_token_matches(
|
||||
prompt: list[int],
|
||||
mm_prompt_updates: "MultiModalPromptUpdates",
|
||||
) -> tuple[list[int], "MultiModalPromptUpdatesApplyResult"]:
|
||||
"""
|
||||
Apply the updates in `mm_prompt_updates` to `prompt`.
|
||||
"""Apply the updates in `mm_prompt_updates` to `prompt`.
|
||||
|
||||
Matches are exclusive even when multiple modalities share
|
||||
the same placeholder tokens. In that case, the modality that
|
||||
@@ -914,8 +902,7 @@ def _iter_placeholders(
|
||||
prompt: list[int],
|
||||
mm_prompt_updates: "MultiModalPromptUpdates",
|
||||
) -> Iterable[PlaceholderFeaturesInfo]:
|
||||
"""
|
||||
Yield each set of placeholder tokens found in `prompt`.
|
||||
"""Yield each set of placeholder tokens found in `prompt`.
|
||||
|
||||
Matches are exclusive even when multiple modalities share
|
||||
the same placeholder tokens. In that case, the modality that
|
||||
@@ -1028,8 +1015,7 @@ class MultiModalProcessingInfo(NamedTuple):
|
||||
|
||||
|
||||
class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
"""
|
||||
Abstract base class to process multi-modal inputs to be used in vLLM.
|
||||
"""Abstract base class to process multi-modal inputs to be used in vLLM.
|
||||
|
||||
Not to be confused with `transformers.ProcessorMixin`.
|
||||
"""
|
||||
@@ -1097,8 +1083,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
out_mm_kwargs: MultiModalKwargsItems,
|
||||
) -> Sequence[PromptUpdate]:
|
||||
"""
|
||||
Given the original multi-modal items for this modality
|
||||
"""Given the original multi-modal items for this modality
|
||||
and HF-processed data, output the updates to perform.
|
||||
|
||||
The information returned by this method is used to update token inputs
|
||||
@@ -1167,8 +1152,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
return processor_data, passthrough_data
|
||||
|
||||
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str | None:
|
||||
"""
|
||||
Get the text to pass to the HF processor alongside the multi-modal
|
||||
"""Get the text to pass to the HF processor alongside the multi-modal
|
||||
data.
|
||||
|
||||
By default, no text is passed. If the HF processor requires that
|
||||
@@ -1183,8 +1167,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
mm_data: Mapping[str, object],
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
) -> tuple[Mapping[str, object], Mapping[str, object]]:
|
||||
"""
|
||||
Pre-process the multi-modal data and HF processor keyword arguments
|
||||
"""Pre-process the multi-modal data and HF processor keyword arguments
|
||||
before they are passed to the HF processor.
|
||||
|
||||
By default, both are returned as-is. If the HF processor expects the
|
||||
@@ -1201,8 +1184,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
processed_data: BatchFeature,
|
||||
) -> BatchFeature:
|
||||
"""
|
||||
Post-process the output of the HF processor.
|
||||
"""Post-process the output of the HF processor.
|
||||
|
||||
By default, the output is returned as-is. If you need to modify the
|
||||
output of the HF processor before it is converted into multi-modal
|
||||
@@ -1215,9 +1197,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
mm_items: MultiModalDataItems,
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
"""
|
||||
Apply the HF processor on the multi-modal data.
|
||||
"""
|
||||
"""Apply the HF processor on the multi-modal data."""
|
||||
valid_mm_items = mm_items.select(
|
||||
{k for k, c in mm_items.get_all_counts().items() if c > 0}
|
||||
)
|
||||
@@ -1250,8 +1230,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
)
|
||||
|
||||
def _postprocess_prompt(self, prompt: list[int]) -> list[int]:
|
||||
"""
|
||||
Post-process the prompt token IDs before locating or applying
|
||||
"""Post-process the prompt token IDs before locating or applying
|
||||
multi-modal prompt updates.
|
||||
|
||||
By default, the prompt is returned as-is. If the HF processor (or
|
||||
@@ -1303,8 +1282,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
cached_update: ResolvedPromptUpdate,
|
||||
new_item_idx: int,
|
||||
) -> ResolvedPromptUpdate:
|
||||
"""
|
||||
Override this if other attributes of `ResolvedPromptUpdate`
|
||||
"""Override this if other attributes of `ResolvedPromptUpdate`
|
||||
also need to be recomputed after retrieving from the cache.
|
||||
"""
|
||||
return replace(cached_update, item_idx=new_item_idx)
|
||||
@@ -1404,8 +1382,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
inputs: ProcessorInputs,
|
||||
timing_ctx: TimingContext,
|
||||
) -> MultiModalProcessingInfo:
|
||||
"""
|
||||
Apply the HF processor on the full prompt text,
|
||||
"""Apply the HF processor on the full prompt text,
|
||||
caching the results and reusing cached results.
|
||||
"""
|
||||
cache = self.cache
|
||||
@@ -1522,8 +1499,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
*,
|
||||
encode_segments_separately: bool = False,
|
||||
) -> tuple[list[int], MultiModalPromptUpdatesApplyResult]:
|
||||
"""
|
||||
Apply multi-modal prompt updates in text space, then encode the
|
||||
"""Apply multi-modal prompt updates in text space, then encode the
|
||||
result.
|
||||
|
||||
A non-special-token target may be tokenized differently inside the
|
||||
@@ -1716,8 +1692,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
inputs: ProcessorInputs,
|
||||
timing_ctx: TimingContext,
|
||||
) -> MultiModalInput:
|
||||
"""
|
||||
Process multi-modal inputs to be used in vLLM.
|
||||
"""Process multi-modal inputs to be used in vLLM.
|
||||
|
||||
The main steps are:
|
||||
|
||||
@@ -1763,8 +1738,7 @@ class EncDecMultiModalProcessor(BaseMultiModalProcessor[_I]):
|
||||
prompt: list[int],
|
||||
mm_items: MultiModalDataItems,
|
||||
) -> list[int]:
|
||||
"""
|
||||
Create input prompt for the encoder. HF processor will be applied on
|
||||
"""Create input prompt for the encoder. HF processor will be applied on
|
||||
this prompt during profiling and generation.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
@@ -1805,8 +1779,7 @@ class EncDecMultiModalProcessor(BaseMultiModalProcessor[_I]):
|
||||
inputs: ProcessorInputs,
|
||||
timing_ctx: TimingContext,
|
||||
) -> MultiModalEncDecInput:
|
||||
"""
|
||||
Process multi-modal inputs to be used in vLLM.
|
||||
"""Process multi-modal inputs to be used in vLLM.
|
||||
The main processing steps are modified to fit encoder-decoder model:
|
||||
1. Create encoder prompt from input prompt text.
|
||||
2. Apply the HF processor on encoder prompt.
|
||||
|
||||
@@ -45,8 +45,7 @@ _I_co = TypeVar("_I_co", bound=BaseProcessingInfo, covariant=True)
|
||||
|
||||
|
||||
class ProcessingInfoFactory(Protocol[_I_co]):
|
||||
"""
|
||||
Constructs a
|
||||
"""Constructs a
|
||||
[`BaseMultiModalProcessor`][vllm.multimodal.processing.BaseMultiModalProcessor]
|
||||
instance from the context.
|
||||
"""
|
||||
@@ -58,8 +57,7 @@ class ProcessingInfoFactory(Protocol[_I_co]):
|
||||
|
||||
|
||||
class DummyInputsBuilderFactory(Protocol[_I]): # type: ignore[misc]
|
||||
"""
|
||||
Constructs a
|
||||
"""Constructs a
|
||||
[`BaseDummyInputsBuilder`][vllm.multimodal.processing.BaseDummyInputsBuilder]
|
||||
instance from the context.
|
||||
"""
|
||||
@@ -68,8 +66,7 @@ class DummyInputsBuilderFactory(Protocol[_I]): # type: ignore[misc]
|
||||
|
||||
|
||||
class MultiModalProcessorFactory(Protocol[_I]): # type: ignore[misc]
|
||||
"""
|
||||
Constructs a
|
||||
"""Constructs a
|
||||
[`BaseMultiModalProcessor`][vllm.multimodal.processing.BaseMultiModalProcessor]
|
||||
instance from the context.
|
||||
"""
|
||||
@@ -101,13 +98,10 @@ class _ProcessorFactories(Generic[_I]):
|
||||
|
||||
|
||||
class MultiModalRegistry:
|
||||
"""
|
||||
A registry that dispatches data processing according to the model.
|
||||
"""
|
||||
"""A registry that dispatches data processing according to the model."""
|
||||
|
||||
def supports_multimodal_inputs(self, model_config: "ModelConfig") -> bool:
|
||||
"""
|
||||
Checks if the model supports multimodal inputs.
|
||||
"""Checks if the model supports multimodal inputs.
|
||||
Returns True if the model is multimodal with any non-zero supported
|
||||
modalities, otherwise returns False, effectively running in
|
||||
text-only mode.
|
||||
@@ -158,8 +152,7 @@ class MultiModalRegistry:
|
||||
info: ProcessingInfoFactory[_I],
|
||||
dummy_inputs: DummyInputsBuilderFactory[_I],
|
||||
):
|
||||
"""
|
||||
Register a multi-modal processor to a model class. The processor
|
||||
"""Register a multi-modal processor to a model class. The processor
|
||||
is constructed lazily, hence a factory method should be passed.
|
||||
|
||||
When the model receives multi-modal data, the provided function is
|
||||
@@ -227,9 +220,7 @@ class MultiModalRegistry:
|
||||
tokenizer: TokenizerLike | None = None,
|
||||
cache: BaseMultiModalProcessorCache | None = None,
|
||||
) -> BaseMultiModalProcessor[BaseProcessingInfo]:
|
||||
"""
|
||||
Create a multi-modal processor for a specific model and tokenizer.
|
||||
"""
|
||||
"""Create a multi-modal processor for a specific model and tokenizer."""
|
||||
if not model_config.is_multimodal_model:
|
||||
model_name = model_config.served_model_name or model_config.model
|
||||
raise ValueError(f"{model_name} is not a multimodal model")
|
||||
@@ -250,8 +241,7 @@ class MultiModalRegistry:
|
||||
processor: BaseMultiModalProcessor | None = None,
|
||||
scheduler_config: "SchedulerConfig | None" = None,
|
||||
) -> MultiModalInput:
|
||||
"""
|
||||
Create dummy data for profiling the memory usage of a model.
|
||||
"""Create dummy data for profiling the memory usage of a model.
|
||||
|
||||
The model is identified by `model_config`.
|
||||
"""
|
||||
|
||||
+16
-17
@@ -67,8 +67,7 @@ def encode_image_base64(
|
||||
image_mode: str | None = "RGB",
|
||||
format: str = "PNG",
|
||||
) -> str:
|
||||
"""
|
||||
Encode a pillow image to base64 format.
|
||||
"""Encode a pillow image to base64 format.
|
||||
|
||||
By default, the image is converted into RGB format before being encoded.
|
||||
Pass `image_mode=None` to keep the original image mode.
|
||||
@@ -83,8 +82,7 @@ def encode_image_url(
|
||||
image_mode: str | None = "RGB",
|
||||
format: str = "PNG",
|
||||
) -> str:
|
||||
"""
|
||||
Encode a pillow image as a data URL.
|
||||
"""Encode a pillow image as a data URL.
|
||||
|
||||
By default, the image is converted into RGB format before being encoded.
|
||||
Pass `image_mode=None` to keep the original image mode.
|
||||
@@ -147,14 +145,14 @@ def get_mm_features_in_window(
|
||||
def argsort_mm_positions(
|
||||
mm_positions: MultiModalPlaceholders,
|
||||
) -> list[tuple[str, int]]:
|
||||
"""
|
||||
Given a `MultiModalPlaceholders`, output a sequence of keys to
|
||||
"""Given a `MultiModalPlaceholders`, output a sequence of keys to
|
||||
sort the dictionary by `offset` (starting index in the input sequence)
|
||||
in ascending order.
|
||||
|
||||
Returns:
|
||||
A list of `(modality, idx)`, which can be used to access an item
|
||||
by `mm_positions[modality][idx]`.
|
||||
|
||||
"""
|
||||
flat_items = (
|
||||
(modality, idx, item)
|
||||
@@ -222,7 +220,8 @@ def strip_covered_mm_data(
|
||||
so the workers never consume the payload fields. M-RoPE models keep
|
||||
CPU-side metadata fields used to compute positions. SHM address items
|
||||
are also kept so workers can balance the sender's reference count. The
|
||||
scheduler-side ``Request`` keeps the full features."""
|
||||
scheduler-side ``Request`` keeps the full features.
|
||||
"""
|
||||
if not mm_features or num_computed_tokens == 0:
|
||||
return mm_features
|
||||
|
||||
@@ -253,8 +252,7 @@ def group_and_batch_mm_items(
|
||||
device: torch.types.Device = None,
|
||||
pin_memory: bool = False,
|
||||
) -> Generator[tuple[int, BatchedTensorInputs]]:
|
||||
"""
|
||||
Group consecutive items (possibly from different requests) into batches.
|
||||
"""Group consecutive items (possibly from different requests) into batches.
|
||||
|
||||
Items must be split across groups if any of the following occurs,
|
||||
as the batch would otherwise be invalid:
|
||||
@@ -270,6 +268,7 @@ def group_and_batch_mm_items(
|
||||
A tuple `(num_items, grouped_kwargs)`, where:
|
||||
- `kwargs` is a dictionary of keyword arguments to pass to the model;
|
||||
- `num_items` is the corresponding number of items.
|
||||
|
||||
"""
|
||||
start_idx = 0
|
||||
for end_idx in range(1, len(items) + 1):
|
||||
@@ -296,8 +295,7 @@ def group_and_batch_mm_kwargs(
|
||||
device: torch.types.Device = None,
|
||||
pin_memory: bool = False,
|
||||
) -> Generator[tuple[str, int, BatchedTensorInputs], None, None]:
|
||||
"""
|
||||
Group consecutive items (possibly from different requests) into batches.
|
||||
"""Group consecutive items (possibly from different requests) into batches.
|
||||
|
||||
Items must be split across groups if any of the following occurs,
|
||||
as the batch would otherwise be invalid:
|
||||
@@ -317,6 +315,7 @@ def group_and_batch_mm_kwargs(
|
||||
- `modality` is the modality of the batch;
|
||||
- `kwargs` is a dictionary of keyword arguments to pass to the model;
|
||||
- `num_items` is the corresponding number of items.
|
||||
|
||||
"""
|
||||
for modality, group in groupby(mm_kwargs, key=lambda x: x[0]):
|
||||
items_lst = [item for _, item in group]
|
||||
@@ -333,14 +332,14 @@ def fetch_audio(
|
||||
audio_url: str,
|
||||
audio_io_kwargs: dict[str, Any] | None = None,
|
||||
) -> tuple[np.ndarray, int | float]:
|
||||
"""
|
||||
Args:
|
||||
"""Args:
|
||||
audio_url: URL of the audio file to fetch.
|
||||
audio_io_kwargs: Additional kwargs passed to handle audio IO.
|
||||
|
||||
Warning:
|
||||
This method has direct access to local files and is only intended
|
||||
to be called by user code. Never call this from the online server!
|
||||
|
||||
"""
|
||||
media_io_kwargs = None if not audio_io_kwargs else {"audio": audio_io_kwargs}
|
||||
media_connector = MediaConnector(
|
||||
@@ -354,14 +353,14 @@ def fetch_image(
|
||||
image_url: str,
|
||||
image_io_kwargs: dict[str, Any] | None = None,
|
||||
) -> Image.Image:
|
||||
"""
|
||||
Args:
|
||||
"""Args:
|
||||
image_url: URL of the image file to fetch.
|
||||
image_io_kwargs: Additional kwargs passed to handle image IO.
|
||||
|
||||
Warning:
|
||||
This method has direct access to local files and is only intended
|
||||
to be called by user code. Never call this from the online server!
|
||||
|
||||
"""
|
||||
media_io_kwargs = None if not image_io_kwargs else {"image": image_io_kwargs}
|
||||
media_connector = MediaConnector(
|
||||
@@ -375,14 +374,14 @@ def fetch_video(
|
||||
video_url: str,
|
||||
video_io_kwargs: dict[str, Any] | None = None,
|
||||
) -> MediaWithBytes[tuple[npt.NDArray, dict[str, Any]]]:
|
||||
"""
|
||||
Args:
|
||||
"""Args:
|
||||
video_url: URL of the video file to fetch.
|
||||
video_io_kwargs: Additional kwargs passed to handle video IO.
|
||||
|
||||
Warning:
|
||||
This method has direct access to local files and is only intended
|
||||
to be called by user code. Never call this from the online server!
|
||||
|
||||
"""
|
||||
media_io_kwargs = None if not video_io_kwargs else {"video": video_io_kwargs}
|
||||
media_connector = MediaConnector(
|
||||
|
||||
@@ -288,6 +288,7 @@ class VideoBackend(VideoLoader):
|
||||
Returns:
|
||||
Tuple of ``(frames, metadata_dict)``, where ``frames`` is a
|
||||
CPU ``np.ndarray`` unless TorchCodec decodes on ``device="cuda"``.
|
||||
|
||||
"""
|
||||
target = VideoTargetMetadata(
|
||||
num_frames=num_frames, fps=fps, max_duration=max_duration
|
||||
@@ -915,8 +916,7 @@ class Molmo2VideoBackend(VideoLoader):
|
||||
sampling_fps: float,
|
||||
max_fps: float = 8.0,
|
||||
) -> list[float]:
|
||||
"""
|
||||
Return the subset of `video_fps` factors that remain multiples
|
||||
"""Return the subset of `video_fps` factors that remain multiples
|
||||
of `sampling_fps`.
|
||||
|
||||
Examples:
|
||||
@@ -931,6 +931,7 @@ class Molmo2VideoBackend(VideoLoader):
|
||||
...
|
||||
ValueError: sampling_fps=2 must divide video_fps=5 to produce
|
||||
consistent frame steps.
|
||||
|
||||
"""
|
||||
if sampling_fps is None:
|
||||
raise ValueError("sampling_fps must be provided")
|
||||
@@ -967,8 +968,8 @@ class Molmo2VideoBackend(VideoLoader):
|
||||
frame_sample_mode: str,
|
||||
candidate_target_fps: list[float],
|
||||
) -> float | None:
|
||||
"""
|
||||
Get the target fps that best spans the videoand has the most frames sampled
|
||||
"""Get the target fps that best spans the video and samples the most
|
||||
frames.
|
||||
"""
|
||||
num_frames_sampled = 0
|
||||
selected_target_fps = None
|
||||
@@ -1272,8 +1273,7 @@ class OpenCVDynamicOpenPanguVideoBackend(VideoLoader):
|
||||
frame_recovery: bool = False,
|
||||
**kwargs,
|
||||
) -> tuple[npt.NDArray, dict[str, Any]]:
|
||||
"""
|
||||
Load video frames with dynamic sampling based on duration.
|
||||
"""Load video frames with dynamic sampling based on duration.
|
||||
|
||||
Args:
|
||||
data: Raw video bytes
|
||||
@@ -1284,6 +1284,7 @@ class OpenCVDynamicOpenPanguVideoBackend(VideoLoader):
|
||||
|
||||
Returns:
|
||||
Tuple of (frames_array, metadata_dict)
|
||||
|
||||
"""
|
||||
# recompute source metadata with adjusted duration to ensure correct
|
||||
# sampling indices computation
|
||||
|
||||
@@ -128,8 +128,7 @@ class OpenCVVideoBackendMixin:
|
||||
frame_indices: list[int],
|
||||
total_frames: int,
|
||||
) -> tuple[npt.NDArray, list[int], dict[int, int]]:
|
||||
"""
|
||||
Read frames with dynamic window forward-scan recovery.
|
||||
"""Read frames with dynamic window forward-scan recovery.
|
||||
|
||||
When a target frame fails to load, the next successfully grabbed
|
||||
frame (before the next target frame) will be used to recover it.
|
||||
@@ -144,6 +143,7 @@ class OpenCVVideoBackendMixin:
|
||||
- frames_array: Array of loaded frames
|
||||
- valid_frame_indices: List of frame indices that were loaded
|
||||
- recovered_map: Dict mapping recovered_idx -> source_idx
|
||||
|
||||
"""
|
||||
width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))
|
||||
height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))
|
||||
|
||||
@@ -16,8 +16,7 @@ import torch
|
||||
def compute_retained_tokens_count(
|
||||
tokens_per_frame: int, num_frames: int, q: float
|
||||
) -> int:
|
||||
"""
|
||||
Compute the number of retained tokens for a given video.
|
||||
"""Compute the number of retained tokens for a given video.
|
||||
Method ensures that we retain all the tokens from the first frame
|
||||
regardless of the pruning rate.
|
||||
|
||||
@@ -28,6 +27,7 @@ def compute_retained_tokens_count(
|
||||
|
||||
Returns:
|
||||
The number of retained tokens.
|
||||
|
||||
"""
|
||||
total_tokens = tokens_per_frame * num_frames
|
||||
evs_num_tokens = int(total_tokens * (1 - q))
|
||||
@@ -41,8 +41,7 @@ def compute_retention_mask(
|
||||
spatial_merge_size: int,
|
||||
q: float,
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Computes the retention mask for input video embeddings.
|
||||
"""Computes the retention mask for input video embeddings.
|
||||
|
||||
Args:
|
||||
video_embeds (`torch.Tensor`): The input video embeddings
|
||||
@@ -55,6 +54,7 @@ def compute_retention_mask(
|
||||
Returns:
|
||||
`torch.Tensor`: The retention mask for the video embeddings of
|
||||
`(T * H * W // spatial_merge_size ^ 2)` shape.
|
||||
|
||||
"""
|
||||
T, H, W = map(int, video_size_thw)
|
||||
|
||||
@@ -98,8 +98,7 @@ def compute_mrope_for_media(
|
||||
tokens_per_second: float = 1.0,
|
||||
video_second_per_grid: float = 1.0,
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Computes the mrope for video embeddings based on the grid dimensions.
|
||||
"""Computes the mrope for video embeddings based on the grid dimensions.
|
||||
Computed mrope positions match original qwen 2.5 implementation,
|
||||
but positions are built for media being the first element in sequence.
|
||||
|
||||
@@ -113,6 +112,7 @@ def compute_mrope_for_media(
|
||||
Tensor of shape `(T * H * W, 4)` where last dimension
|
||||
represents mrope positions [0:3), while the last channel
|
||||
contains value of llm_grid_w repeated for all positions.
|
||||
|
||||
"""
|
||||
llm_grid_t = video_size_thw[0]
|
||||
llm_grid_h = video_size_thw[1] // spatial_merge_size
|
||||
@@ -160,8 +160,7 @@ def recompute_mrope_positions(
|
||||
image_token_id: int,
|
||||
video_token_id: int,
|
||||
) -> tuple[torch.LongTensor, int]:
|
||||
"""
|
||||
Update part of input mrope positions.
|
||||
"""Update part of input mrope positions.
|
||||
Original mrope_positions are computed incorrectly, so once we prune media
|
||||
tokens we should reflect this in the mrope positions for the LLM.
|
||||
|
||||
@@ -204,8 +203,8 @@ def recompute_mrope_positions(
|
||||
|
||||
Returns:
|
||||
Tuple of (mrope_positions, mrope_position_delta).
|
||||
"""
|
||||
|
||||
"""
|
||||
# Tensors
|
||||
positions: torch.LongTensor = typing.cast(
|
||||
torch.LongTensor, mrope_positions.clone()
|
||||
|
||||
@@ -48,6 +48,7 @@ def compute_retention_mask(
|
||||
Flat bool tensor of shape `(T*H*W/merge^2,)`, True for retained
|
||||
tokens. The True count equals `compute_retained_tokens_count` so
|
||||
placeholders sized at prompt-processing time match exactly.
|
||||
|
||||
"""
|
||||
T, H, W = map(int, video_size_thw)
|
||||
rows = H // spatial_merge_size
|
||||
|
||||
Reference in New Issue
Block a user