Convert formatting to use ruff instead of yapf + isort (#26247)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
Harry Mellor
2025-10-05 07:06:22 -07:00
committed by GitHub
parent 17edd8a807
commit d6953beb91
1508 changed files with 115205 additions and 94107 deletions
+11 -4
View File
@@ -1,10 +1,17 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from .hasher import MultiModalHasher
from .inputs import (BatchedTensorInputs, ModalityData, MultiModalDataBuiltins,
MultiModalDataDict, MultiModalKwargs,
MultiModalKwargsItems, MultiModalPlaceholderDict,
MultiModalUUIDDict, NestedTensors)
from .inputs import (
BatchedTensorInputs,
ModalityData,
MultiModalDataBuiltins,
MultiModalDataDict,
MultiModalKwargs,
MultiModalKwargsItems,
MultiModalPlaceholderDict,
MultiModalUUIDDict,
NestedTensors,
)
from .registry import MultiModalRegistry
MULTIMODAL_REGISTRY = MultiModalRegistry()
+14 -12
View File
@@ -66,23 +66,25 @@ class AudioResampler:
orig_sr: float,
) -> npt.NDArray[np.floating]:
if self.target_sr is None:
raise RuntimeError("Audio resampling is not supported when "
"`target_sr` is not provided")
raise RuntimeError(
"Audio resampling is not supported when `target_sr` is not provided"
)
if self.method == "librosa":
return resample_audio_librosa(audio,
orig_sr=orig_sr,
target_sr=self.target_sr)
return resample_audio_librosa(
audio, orig_sr=orig_sr, target_sr=self.target_sr
)
elif self.method == "scipy":
return resample_audio_scipy(audio,
orig_sr=orig_sr,
target_sr=self.target_sr)
return resample_audio_scipy(
audio, orig_sr=orig_sr, target_sr=self.target_sr
)
else:
raise ValueError(f"Invalid resampling method: {self.method}. "
"Supported methods are 'librosa' and 'scipy'.")
raise ValueError(
f"Invalid resampling method: {self.method}. "
"Supported methods are 'librosa' and 'scipy'."
)
class AudioMediaIO(MediaIO[tuple[npt.NDArray, float]]):
def __init__(self, **kwargs) -> None:
super().__init__()
@@ -113,4 +115,4 @@ class AudioMediaIO(MediaIO[tuple[npt.NDArray, float]]):
soundfile.write(buffer, audio, sr, format="WAV")
data = buffer.getvalue()
return base64.b64encode(data).decode('utf-8')
return base64.b64encode(data).decode("utf-8")
-1
View File
@@ -9,7 +9,6 @@ _T = TypeVar("_T")
class MediaIO(ABC, Generic[_T]):
@abstractmethod
def load_bytes(self, data: bytes) -> _T:
raise NotImplementedError
+52 -40
View File
@@ -11,17 +11,24 @@ import torch
from typing_extensions import TypeAlias, override
from vllm.distributed.device_communicators.shm_object_storage import (
MsgpackSerde, SingleWriterShmObjectStorage, SingleWriterShmRingBuffer)
MsgpackSerde,
SingleWriterShmObjectStorage,
SingleWriterShmRingBuffer,
)
from vllm.envs import VLLM_OBJECT_STORAGE_SHM_BUFFER_NAME
from vllm.logger import init_logger
from vllm.utils import GiB_bytes, LRUCache, MiB_bytes
from vllm.utils.jsontree import (json_count_leaves, json_map_leaves,
json_reduce_leaves)
from vllm.utils.jsontree import json_count_leaves, json_map_leaves, json_reduce_leaves
from .inputs import (MultiModalBatchedField, MultiModalFeatureSpec,
MultiModalFieldElem, MultiModalKwargs,
MultiModalKwargsItem, MultiModalKwargsItems,
NestedTensors)
from .inputs import (
MultiModalBatchedField,
MultiModalFeatureSpec,
MultiModalFieldElem,
MultiModalKwargs,
MultiModalKwargsItem,
MultiModalKwargsItems,
NestedTensors,
)
if TYPE_CHECKING:
from vllm.config import ModelConfig, VllmConfig
@@ -90,7 +97,6 @@ _V = TypeVar("_V", bound=MultiModalCacheValue)
class MultiModalCache:
@classmethod
def get_leaf_size(cls, leaf: object) -> int:
if isinstance(leaf, MultiModalProcessorCacheItem):
@@ -99,8 +105,15 @@ class MultiModalCache:
return leaf.item_size
# These are not subclasses of dict
if isinstance(leaf, (MultiModalKwargs, MultiModalKwargsItems,
MultiModalKwargsItem, MultiModalFieldElem)):
if isinstance(
leaf,
(
MultiModalKwargs,
MultiModalKwargsItems,
MultiModalKwargsItem,
MultiModalFieldElem,
),
):
return cls.get_item_size(leaf.data) # type: ignore
# sys.getsizeof doesn't work for tensors
@@ -116,8 +129,9 @@ class MultiModalCache:
*,
debug: bool = False,
) -> int:
size = json_reduce_leaves(operator.add,
json_map_leaves(cls.get_leaf_size, value))
size = json_reduce_leaves(
operator.add, json_map_leaves(cls.get_leaf_size, value)
)
if debug:
leaf_count = json_count_leaves(value)
@@ -241,17 +255,19 @@ class BaseMultiModalCache(ABC, Generic[_I, _O]):
raise NotImplementedError
MultiModalProcessorCacheInItem: TypeAlias = \
Optional[tuple[MultiModalKwargsItem, Sequence["ResolvedPromptUpdate"]]]
MultiModalProcessorCacheInItem: TypeAlias = Optional[
tuple[MultiModalKwargsItem, Sequence["ResolvedPromptUpdate"]]
]
MultiModalProcessorCacheOutItem: TypeAlias = \
tuple[Optional[MultiModalKwargsItem], Sequence["ResolvedPromptUpdate"]]
MultiModalProcessorCacheOutItem: TypeAlias = tuple[
Optional[MultiModalKwargsItem], Sequence["ResolvedPromptUpdate"]
]
class BaseMultiModalProcessorCache(
BaseMultiModalCache[MultiModalProcessorCacheInItem,
MultiModalProcessorCacheOutItem]):
BaseMultiModalCache[MultiModalProcessorCacheInItem, MultiModalProcessorCacheOutItem]
):
"""The required interface for caches on P0."""
@abstractmethod
@@ -405,15 +421,13 @@ class ShmObjectStoreSenderCache(BaseMultiModalProcessorCache):
create=True, # sender is the writer
)
self._shm_cache = SingleWriterShmObjectStorage(
max_object_size=mm_config.mm_shm_cache_max_object_size_mb *
MiB_bytes,
max_object_size=mm_config.mm_shm_cache_max_object_size_mb * MiB_bytes,
n_readers=self.world_size,
ring_buffer=ring_buffer,
serde_class=MsgpackSerde,
)
# cache (prompt_updates, modality) for P0 only
self._p0_cache: dict[str, tuple[Sequence[ResolvedPromptUpdate],
str]] = {}
self._p0_cache: dict[str, tuple[Sequence[ResolvedPromptUpdate], str]] = {}
@override
def is_cached_item(self, mm_hash: str) -> bool:
@@ -425,12 +439,10 @@ class ShmObjectStoreSenderCache(BaseMultiModalProcessorCache):
mm_item: MultiModalProcessorCacheInItem,
mm_hash: str,
) -> MultiModalProcessorCacheOutItem:
if self._shm_cache.is_cached(mm_hash):
address, monotonic_id = self._shm_cache.get_cached(mm_hash)
prompt_updates, modality = self._p0_cache[mm_hash]
return self.address_as_item(address, monotonic_id,
modality), prompt_updates
return self.address_as_item(address, monotonic_id, modality), prompt_updates
assert mm_item is not None, f"Expected a cached item for {mm_hash=}"
@@ -440,15 +452,15 @@ class ShmObjectStoreSenderCache(BaseMultiModalProcessorCache):
if len(self._p0_cache) >= 2 * len(self._shm_cache.key_index):
self.remove_dangling_items()
self._p0_cache[mm_hash] = mm_item[1], mm_item[0].modality
address_item = self.address_as_item(address, monotonic_id,
mm_item[0].modality)
address_item = self.address_as_item(
address, monotonic_id, mm_item[0].modality
)
return address_item, mm_item[1]
except (ValueError, MemoryError) as e:
# put may fail if the object is too large or
# the cache is full.
# In this case we log the error and keep the original mm_input.
logger.debug("Failed to cache mm_input with hash %s: %s", mm_hash,
e)
logger.debug("Failed to cache mm_input with hash %s: %s", mm_hash, e)
return mm_item
@override
@@ -463,8 +475,9 @@ class ShmObjectStoreSenderCache(BaseMultiModalProcessorCache):
for mm_hash in dangling_hashes:
del self._p0_cache[mm_hash]
def address_as_item(self, address: int, monotonic_id: int,
modality: str) -> MultiModalKwargsItem:
def address_as_item(
self, address: int, monotonic_id: int, modality: str
) -> MultiModalKwargsItem:
addr_elem = MultiModalFieldElem(
modality=modality,
key="address",
@@ -494,9 +507,10 @@ def _enable_processor_cache(
def _enable_ipc_cache(vllm_config: "VllmConfig") -> bool:
parallel_config = vllm_config.parallel_config
supports_ipc_cache = ((parallel_config._api_process_count == 1
and parallel_config.data_parallel_size == 1)
or parallel_config.data_parallel_external_lb)
supports_ipc_cache = (
parallel_config._api_process_count == 1
and parallel_config.data_parallel_size == 1
) or parallel_config.data_parallel_external_lb
return supports_ipc_cache
@@ -542,8 +556,8 @@ def processor_only_cache_from_config(
class BaseMultiModalReceiverCache(
BaseMultiModalCache[Optional[MultiModalKwargsItem],
MultiModalKwargsItem]):
BaseMultiModalCache[Optional[MultiModalKwargsItem], MultiModalKwargsItem]
):
"""The required interface for caches on P1."""
def get_and_update_features(
@@ -552,8 +566,7 @@ class BaseMultiModalReceiverCache(
) -> list["MultiModalFeatureSpec"]:
"""Update multimodal features with cached encoder outputs."""
for feature in mm_features:
feature.data = self.get_and_update_item(feature.data,
feature.identifier)
feature.data = self.get_and_update_item(feature.data, feature.identifier)
return mm_features
@@ -623,8 +636,7 @@ class ShmObjectStoreReceiverCache(BaseMultiModalReceiverCache):
create=False, # Server is a reader
)
self._shm_cache = SingleWriterShmObjectStorage(
max_object_size=mm_config.mm_shm_cache_max_object_size_mb *
MiB_bytes,
max_object_size=mm_config.mm_shm_cache_max_object_size_mb * MiB_bytes,
n_readers=self.world_size,
ring_buffer=ring_buffer,
serde_class=MsgpackSerde,
+61 -40
View File
@@ -13,8 +13,9 @@ import typing
import torch
def compute_retained_tokens_count(video_size_thw: torch.LongTensor,
spatial_merge_size: int, q: float) -> int:
def compute_retained_tokens_count(
video_size_thw: torch.LongTensor, spatial_merge_size: int, q: float
) -> int:
"""
Compute the number of retained tokens for a given video.
Method ensures that we retain all the tokens from the first frame
@@ -66,23 +67,21 @@ def compute_retention_mask(
)
# Core EVS
similarity = torch.nn.functional.cosine_similarity(video_embeds[1:, ...],
video_embeds[:-1, ...],
dim=-1)
similarity = torch.nn.functional.cosine_similarity(
video_embeds[1:, ...], video_embeds[:-1, ...], dim=-1
)
dissimilarity = 1 - similarity
# Always ensure we include all tokens from the first frame
dissimilarity = torch.cat(
[255 * torch.ones_like(video_embeds[:1, :, :, 0]), dissimilarity],
dim=0)
[255 * torch.ones_like(video_embeds[:1, :, :, 0]), dissimilarity], dim=0
)
dissimilarity_flat = dissimilarity.view(-1)
order = torch.argsort(dissimilarity_flat,
dim=-1,
descending=True,
stable=True)
retain_num_tokens = compute_retained_tokens_count(video_size_thw,
spatial_merge_size, q)
order = torch.argsort(dissimilarity_flat, dim=-1, descending=True, stable=True)
retain_num_tokens = compute_retained_tokens_count(
video_size_thw, spatial_merge_size, q
)
topk_indices = order[:retain_num_tokens]
retention_mask = torch.zeros_like(dissimilarity_flat, dtype=torch.bool)
@@ -119,18 +118,34 @@ def compute_mrope_for_media(
llm_grid_h = video_size_thw[1] // spatial_merge_size
llm_grid_w = video_size_thw[2] // spatial_merge_size
t_index = ((torch.arange(llm_grid_t).view(-1, 1).expand(
-1, llm_grid_h * llm_grid_w).mul(
tokens_per_second * video_second_per_grid)).long().flatten())
h_index = (torch.arange(llm_grid_h).view(1, -1,
1).expand(llm_grid_t, -1,
llm_grid_w).flatten())
w_index = (torch.arange(llm_grid_w).view(1, 1, -1).expand(
llm_grid_t, llm_grid_h, -1).flatten())
llm_grid_w = (torch.tensor([llm_grid_w
]).view(1, 1,
1).expand(llm_grid_t, llm_grid_h,
llm_grid_w).flatten())
t_index = (
(
torch.arange(llm_grid_t)
.view(-1, 1)
.expand(-1, llm_grid_h * llm_grid_w)
.mul(tokens_per_second * video_second_per_grid)
)
.long()
.flatten()
)
h_index = (
torch.arange(llm_grid_h)
.view(1, -1, 1)
.expand(llm_grid_t, -1, llm_grid_w)
.flatten()
)
w_index = (
torch.arange(llm_grid_w)
.view(1, 1, -1)
.expand(llm_grid_t, llm_grid_h, -1)
.flatten()
)
llm_grid_w = (
torch.tensor([llm_grid_w])
.view(1, 1, 1)
.expand(llm_grid_t, llm_grid_h, llm_grid_w)
.flatten()
)
positions = torch.stack([t_index, h_index, w_index, llm_grid_w], dim=1)
return positions
@@ -183,7 +198,8 @@ def recompute_mrope_positions(
# Tensors
positions: torch.LongTensor = typing.cast(
torch.LongTensor, mrope_positions.clone()) # (3, N)
torch.LongTensor, mrope_positions.clone()
) # (3, N)
N = input_ids.numel()
image_mask = input_ids.eq(image_token_id)
@@ -193,8 +209,7 @@ def recompute_mrope_positions(
# Early exit: no media in this chunk
if len(multimodal_positions) == 0:
delta = (int((positions.max().item() + 1) -
N) if positions.numel() else -N)
delta = int((positions.max().item() + 1) - N) if positions.numel() else -N
return positions, delta
total_mm_tokens = torch.count_nonzero(media_mask)
@@ -203,12 +218,12 @@ def recompute_mrope_positions(
# Early exit: we've updated positions for all media tokens
# (and consequently - for all remaining text tokens)
if seen_mm_tokens == total_mm_tokens:
delta = (int((positions.max().item() + 1) -
N) if positions.numel() else -N)
delta = int((positions.max().item() + 1) - N) if positions.numel() else -N
return positions, delta
vision_start_indices = (input_ids == vision_start_token_id).nonzero(
as_tuple=True)[0]
vision_start_indices = (input_ids == vision_start_token_id).nonzero(as_tuple=True)[
0
]
for mm_pos in multimodal_positions:
# Each mm_pos can be a complete embedding for single media
@@ -218,8 +233,9 @@ def recompute_mrope_positions(
# - Current prefill chunk has no vision start indexes at all
# - Vision start token appeared in previous prefill round
# - Regular case
seen_vision_start_indices = vision_start_indices[vision_start_indices <
num_computed_tokens]
seen_vision_start_indices = vision_start_indices[
vision_start_indices < num_computed_tokens
]
if len(seen_vision_start_indices):
# If we have encountered some vision start indexes,
@@ -228,19 +244,23 @@ def recompute_mrope_positions(
# | TTTTTTTTTSVVVVVVVVVV|VVVVVVTTTTTTTTTTTTTTTT|
last_vision_start_token = seen_vision_start_indices[-1]
seem_mm_tokens_before_last_vision_start = torch.count_nonzero(
media_mask[:last_vision_start_token])
media_mask[:last_vision_start_token]
)
in_the_middle_of_media = (
seen_mm_tokens > seem_mm_tokens_before_last_vision_start)
seen_mm_tokens > seem_mm_tokens_before_last_vision_start
)
if in_the_middle_of_media:
mm_embeddings_seen = (seen_mm_tokens -
seem_mm_tokens_before_last_vision_start)
mm_embeddings_seen = (
seen_mm_tokens - seem_mm_tokens_before_last_vision_start
)
global_mm_start = last_vision_start_token
else:
# We have completed previous mm_embedding part and
# ready to start a new one
next_vision_start_token = vision_start_indices[
vision_start_indices >= num_computed_tokens][0]
vision_start_indices >= num_computed_tokens
][0]
mm_embeddings_seen = 0
global_mm_start = next_vision_start_token
@@ -248,7 +268,8 @@ def recompute_mrope_positions(
# If there were no vision start indexes so far,
# let's find first vision start index
next_vision_start_token = vision_start_indices[
vision_start_indices >= num_computed_tokens][0]
vision_start_indices >= num_computed_tokens
][0]
mm_embeddings_seen = 0
global_mm_start = next_vision_start_token
+25 -20
View File
@@ -17,23 +17,23 @@ logger = init_logger(__name__)
class MultiModalHasher:
@classmethod
def serialize_item(cls, obj: object) -> Iterable[Union[bytes, memoryview]]:
# Simple cases
if isinstance(obj, (bytes, memoryview)):
return (obj, )
return (obj,)
if isinstance(obj, str):
return (obj.encode("utf-8"), )
return (obj.encode("utf-8"),)
if isinstance(obj, (int, float)):
return (np.array(obj).tobytes(), )
return (np.array(obj).tobytes(),)
if isinstance(obj, Image.Image):
exif = obj.getexif()
if Image.ExifTags.Base.ImageID in exif and isinstance(
exif[Image.ExifTags.Base.ImageID], uuid.UUID):
exif[Image.ExifTags.Base.ImageID], uuid.UUID
):
# If the image has exif ImageID tag, use that
return (exif[Image.ExifTags.Base.ImageID].bytes, )
return (exif[Image.ExifTags.Base.ImageID].bytes,)
data = {"mode": obj.mode, "data": np.asarray(obj)}
if obj.palette is not None:
data["palette"] = obj.palette.palette
@@ -49,30 +49,35 @@ class MultiModalHasher:
# Workaround: View the tensor as a contiguous 1D array of bytes
if tensor_dtype == torch.bfloat16:
tensor_obj = tensor_obj.contiguous()
tensor_obj = tensor_obj.view(
(tensor_obj.numel(), )).view(torch.uint8)
tensor_obj = tensor_obj.view((tensor_obj.numel(),)).view(torch.uint8)
return cls.iter_item_to_bytes(
"tensor", {
"tensor",
{
"original_dtype": str(tensor_dtype),
"original_shape": tuple(tensor_shape),
"data": tensor_obj.numpy(),
})
},
)
return cls.iter_item_to_bytes("tensor", tensor_obj.numpy())
if isinstance(obj, np.ndarray):
# If the array is non-contiguous, we need to copy it first
arr_data = obj.view(
np.uint8).data if obj.flags.c_contiguous else obj.tobytes()
return cls.iter_item_to_bytes("ndarray", {
"dtype": obj.dtype.str,
"shape": obj.shape,
"data": arr_data,
})
arr_data = (
obj.view(np.uint8).data if obj.flags.c_contiguous else obj.tobytes()
)
return cls.iter_item_to_bytes(
"ndarray",
{
"dtype": obj.dtype.str,
"shape": obj.shape,
"data": arr_data,
},
)
logger.warning(
"No serialization method found for %s. "
"Falling back to pickle.", type(obj))
"No serialization method found for %s. Falling back to pickle.", type(obj)
)
return (pickle.dumps(obj), )
return (pickle.dumps(obj),)
@classmethod
def iter_item_to_bytes(
+14 -13
View File
@@ -12,9 +12,9 @@ from PIL import Image
from .base import MediaIO
def rescale_image_size(image: Image.Image,
size_factor: float,
transpose: int = -1) -> Image.Image:
def rescale_image_size(
image: Image.Image, size_factor: float, transpose: int = -1
) -> Image.Image:
"""Rescale the dimensions of an image by a constant factor."""
new_width = int(image.width * size_factor)
new_height = int(image.height * size_factor)
@@ -26,7 +26,7 @@ def rescale_image_size(image: Image.Image,
def rgba_to_rgb(
image: Image.Image,
background_color: Union[tuple[int, int, int], list[int]] = (255, 255, 255)
background_color: Union[tuple[int, int, int], list[int]] = (255, 255, 255),
) -> Image.Image:
"""Convert an RGBA image to RGB with filled background color."""
assert image.mode == "RGBA"
@@ -45,7 +45,6 @@ def convert_image_mode(image: Image.Image, to_mode: str):
class ImageMediaIO(MediaIO[Image.Image]):
def __init__(self, image_mode: str = "RGB", **kwargs) -> None:
super().__init__()
@@ -59,18 +58,21 @@ class ImageMediaIO(MediaIO[Image.Image]):
# Extract RGBA background color from kwargs if provided
# Default to white background for backward compatibility
rgba_bg = kwargs.get('rgba_background_color', (255, 255, 255))
rgba_bg = kwargs.get("rgba_background_color", (255, 255, 255))
# Convert list to tuple for consistency
if isinstance(rgba_bg, list):
rgba_bg = tuple(rgba_bg)
# Validate rgba_background_color format
if not (isinstance(rgba_bg, tuple) and len(rgba_bg) == 3
and all(isinstance(c, int) and 0 <= c <= 255
for c in rgba_bg)):
if not (
isinstance(rgba_bg, tuple)
and len(rgba_bg) == 3
and all(isinstance(c, int) and 0 <= c <= 255 for c in rgba_bg)
):
raise ValueError(
"rgba_background_color must be a list or tuple of 3 integers "
"in the range [0, 255].")
"in the range [0, 255]."
)
self.rgba_background_color = rgba_bg
def _convert_image_mode(self, image: Image.Image) -> Image.Image:
@@ -108,11 +110,10 @@ class ImageMediaIO(MediaIO[Image.Image]):
image.save(buffer, image_format)
data = buffer.getvalue()
return pybase64.b64encode(data).decode('utf-8')
return pybase64.b64encode(data).decode("utf-8")
class ImageEmbeddingMediaIO(MediaIO[torch.Tensor]):
def __init__(self) -> None:
super().__init__()
@@ -127,4 +128,4 @@ class ImageEmbeddingMediaIO(MediaIO[torch.Tensor]):
return torch.load(filepath, weights_only=True)
def encode_base64(self, media: torch.Tensor) -> str:
return pybase64.b64encode(media.numpy()).decode('utf-8')
return pybase64.b64encode(media.numpy()).decode("utf-8")
+106 -78
View File
@@ -7,8 +7,7 @@ from collections.abc import Mapping, Sequence
from dataclasses import dataclass
from functools import partial
from itertools import accumulate
from typing import (TYPE_CHECKING, Any, Literal, Optional, TypedDict, Union,
cast, final)
from typing import TYPE_CHECKING, Any, Literal, Optional, TypedDict, Union, cast, final
import numpy as np
from typing_extensions import NotRequired, TypeAlias, TypeVar, deprecated
@@ -35,8 +34,9 @@ A `transformers.image_utils.ImageInput` representing a single image
item, which can be passed to a HuggingFace `ImageProcessor`.
"""
HfVideoItem: TypeAlias = Union[list["Image"], np.ndarray, "torch.Tensor",
list[np.ndarray], list["torch.Tensor"]]
HfVideoItem: TypeAlias = Union[
list["Image"], np.ndarray, "torch.Tensor", list[np.ndarray], list["torch.Tensor"]
]
"""
A `transformers.image_utils.VideoInput` representing a single video
item, which can be passed to a HuggingFace `VideoProcessor`.
@@ -58,8 +58,9 @@ which are treated as image embeddings;
these are directly passed to the model without HF processing.
"""
VideoItem: TypeAlias = Union[HfVideoItem, "torch.Tensor",
tuple[HfVideoItem, dict[str, Any]]]
VideoItem: TypeAlias = Union[
HfVideoItem, "torch.Tensor", tuple[HfVideoItem, dict[str, Any]]
]
"""
A `transformers.video_utils.VideoInput` representing a single video item.
This can be passed to a HuggingFace `VideoProcessor`
@@ -70,8 +71,7 @@ which are treated as video embeddings;
these are directly passed to the model without HF processing.
"""
AudioItem: TypeAlias = Union[HfAudioItem, tuple[np.ndarray, float],
"torch.Tensor"]
AudioItem: TypeAlias = Union[HfAudioItem, tuple[np.ndarray, float], "torch.Tensor"]
"""
Represents a single audio
item, which can be passed to a HuggingFace `AudioProcessor`.
@@ -177,8 +177,12 @@ class PlaceholderRange:
return nested_tensors_equal(self.is_embed, other.is_embed)
NestedTensors: TypeAlias = Union[list["NestedTensors"], list["torch.Tensor"],
"torch.Tensor", tuple["torch.Tensor", ...]]
NestedTensors: TypeAlias = Union[
list["NestedTensors"],
list["torch.Tensor"],
"torch.Tensor",
tuple["torch.Tensor", ...],
]
"""
Uses a list instead of a tensor if the dimensions of each element do not match.
"""
@@ -193,11 +197,13 @@ def nested_tensors_equal(a: NestedTensors, b: NestedTensors) -> bool:
return isinstance(a, torch.Tensor) and torch.equal(b, a)
if isinstance(a, list):
return (isinstance(b, list)
and all(nested_tensors_equal(a_, b_) for a_, b_ in zip(a, b)))
return isinstance(b, list) and all(
nested_tensors_equal(a_, b_) for a_, b_ in zip(a, b)
)
if isinstance(b, list):
return (isinstance(a, list)
and all(nested_tensors_equal(b_, a_) for b_, a_ in zip(b, a)))
return isinstance(a, list) and all(
nested_tensors_equal(b_, a_) for b_, a_ in zip(b, a)
)
# Both a and b are scalars
return a == b
@@ -214,7 +220,7 @@ A dictionary containing nested tensors which have been batched via
class MultiModalFeatureSpec:
"""
Represents a single multimodal input with its processed data and metadata.
Used by the V1 engine to track multimodal data through processing and
caching. A request containing multiple multimodal items will have one
MultiModalFeatureSpec per item.
@@ -280,9 +286,11 @@ class MultiModalFieldElem:
else:
data_equal = nested_tensors_equal(self.data, other.data)
return ((self.modality, self.key) == (other.modality, other.key)
and data_equal
and type(self.field) == type(other.field)) # noqa: E721
return (
(self.modality, self.key) == (other.modality, other.key)
and data_equal
and type(self.field) == type(other.field)
) # noqa: E721
@dataclass(frozen=True)
@@ -385,10 +393,12 @@ class MultiModalBatchedField(BaseMultiModalField):
return batch[0].unsqueeze(0).contiguous()
first_shape = batch[0].shape
if all(elem.shape == first_shape for elem in batch):
out = torch.empty((len(batch), *batch[0].shape),
dtype=batch[0].dtype,
device=batch[0].device,
pin_memory=pin_memory)
out = torch.empty(
(len(batch), *batch[0].shape),
dtype=batch[0].dtype,
device=batch[0].device,
pin_memory=pin_memory,
)
return torch.stack(batch, out=out)
return batch
@@ -401,6 +411,7 @@ class MultiModalFlatField(BaseMultiModalField):
[`MultiModalFieldConfig.flat`][vllm.multimodal.inputs.MultiModalFieldConfig.flat]
[`MultiModalFieldConfig.flat_from_sizes`][vllm.multimodal.inputs.MultiModalFieldConfig.flat_from_sizes]
"""
slices: Union[Sequence[slice], Sequence[Sequence[slice]]]
dim: int = 0
@@ -412,8 +423,9 @@ class MultiModalFlatField(BaseMultiModalField):
) -> Sequence[MultiModalFieldElem]:
field_factory = self._field_factory(modality=modality, key=key)
if not is_list_of(self.slices, slice, check="all"):
assert isinstance(data, torch.Tensor), \
assert isinstance(data, torch.Tensor), (
"torch.Tensor is required for multiple slices"
)
return [field_factory(data[cast(slice, s)]) for s in self.slices]
def _reduce_data(
@@ -433,17 +445,19 @@ class MultiModalFlatField(BaseMultiModalField):
dim = self.dim + (self.dim < 0) * len(batch[0].shape)
def _shape_before_after(tensor: torch.Tensor):
return tensor.shape[:dim], tensor.shape[dim + 1:]
return tensor.shape[:dim], tensor.shape[dim + 1 :]
first_shape = _shape_before_after(batch[0])
if all(_shape_before_after(elem) == first_shape for elem in batch):
shape_before, shape_after = first_shape
shape_concat = sum(item.shape[dim] for item in batch)
out = torch.empty((*shape_before, shape_concat, *shape_after),
dtype=batch[0].dtype,
device=batch[0].device,
pin_memory=pin_memory)
out = torch.empty(
(*shape_before, shape_concat, *shape_after),
dtype=batch[0].dtype,
device=batch[0].device,
pin_memory=pin_memory,
)
return torch.concat(batch, dim=self.dim, out=out)
assert self.dim == 0, "dim == 0 is required for nested list"
@@ -456,6 +470,7 @@ class MultiModalSharedField(BaseMultiModalField):
Info:
[`MultiModalFieldConfig.shared`][vllm.multimodal.inputs.MultiModalFieldConfig.shared]
"""
batch_size: int
def build_elems(
@@ -477,7 +492,6 @@ class MultiModalSharedField(BaseMultiModalField):
class MultiModalFieldConfig:
@staticmethod
def batched(modality: str):
"""
@@ -508,9 +522,11 @@ class MultiModalFieldConfig:
)
@staticmethod
def flat(modality: str,
slices: Union[Sequence[slice], Sequence[Sequence[slice]]],
dim: int = 0):
def flat(
modality: str,
slices: Union[Sequence[slice], Sequence[Sequence[slice]]],
dim: int = 0,
):
"""
Defines a field where an element in the batch is obtained by
slicing along the first dimension of the underlying data.
@@ -561,9 +577,7 @@ class MultiModalFieldConfig:
)
@staticmethod
def flat_from_sizes(modality: str,
size_per_item: "torch.Tensor",
dim: int = 0):
def flat_from_sizes(modality: str, size_per_item: "torch.Tensor", dim: int = 0):
"""
Defines a field where an element in the batch is obtained by
slicing along the first dimension of the underlying data.
@@ -609,13 +623,17 @@ class MultiModalFieldConfig:
"""
if size_per_item.ndim != 1:
raise ValueError("size_per_item should be a 1-D tensor, "
f"but found shape: {size_per_item.shape}")
raise ValueError(
"size_per_item should be a 1-D tensor, "
f"but found shape: {size_per_item.shape}"
)
slice_idxs = [0, *accumulate(size_per_item)]
slices = [(slice(None, None, None), ) * dim +
(slice(slice_idxs[i], slice_idxs[i + 1]), )
for i in range(len(size_per_item))]
slices = [
(slice(None, None, None),) * dim
+ (slice(slice_idxs[i], slice_idxs[i + 1]),)
for i in range(len(size_per_item))
]
return MultiModalFieldConfig.flat(modality, slices, dim=dim)
@@ -745,7 +763,8 @@ class MultiModalKwargsItems(UserDict[str, Sequence[_I]]):
if len(set(batch_sizes.values())) > 1:
raise ValueError(
f"Cannot merge different batch sizes for {modality=}! "
f"Found: {batch_sizes=}")
f"Found: {batch_sizes=}"
)
batch_size = next(iter(batch_sizes.values()))
for item_idx in range(batch_size):
@@ -761,8 +780,10 @@ class MultiModalKwargsItems(UserDict[str, Sequence[_I]]):
def __getitem__(self, modality: str) -> Sequence[_I]:
if modality not in self:
raise KeyError(f"Modality {modality!r} not found. "
f"Available modalities: {set(self.keys())}")
raise KeyError(
f"Modality {modality!r} not found. "
f"Available modalities: {set(self.keys())}"
)
return super().__getitem__(modality) # type: ignore[return-value]
@@ -770,8 +791,7 @@ class MultiModalKwargsItems(UserDict[str, Sequence[_I]]):
for modality, items in self.items():
for i, item in enumerate(items):
if item is None:
raise RuntimeError(
f"Found empty mm_items[{modality}][{i}]")
raise RuntimeError(f"Found empty mm_items[{modality}][{i}]")
return self # type: ignore[return-value]
@@ -780,17 +800,19 @@ class MultiModalKwargsItems(UserDict[str, Sequence[_I]]):
for modality, items in self.items():
for i, item in enumerate(items):
if item is None:
raise RuntimeError("Cannot build data from empty "
f"mm_items[{modality}][{i}]")
raise RuntimeError(
f"Cannot build data from empty mm_items[{modality}][{i}]"
)
for key, elem in item.items():
elems_by_key[key].append(elem)
return MultiModalKwargs({
key:
elems[0].field.reduce_data(elems, pin_memory=pin_memory)
for key, elems in elems_by_key.items()
})
return MultiModalKwargs(
{
key: elems[0].field.reduce_data(elems, pin_memory=pin_memory)
for key, elems in elems_by_key.items()
}
)
MultiModalKwargsOptionalItems: TypeAlias = Union[
@@ -806,33 +828,36 @@ class MultiModalKwargs(UserDict[str, NestedTensors]):
"""
@staticmethod
@deprecated("`MultiModalKwargs.from_hf_inputs` is deprecated and "
"will be removed in v0.13. "
"Please use `MultiModalKwargsItems.from_hf_inputs` and "
"access the tensor data using `.get_data()`.")
@deprecated(
"`MultiModalKwargs.from_hf_inputs` is deprecated and "
"will be removed in v0.13. "
"Please use `MultiModalKwargsItems.from_hf_inputs` and "
"access the tensor data using `.get_data()`."
)
def from_hf_inputs(
hf_inputs: "BatchFeature",
config_by_key: Mapping[str, MultiModalFieldConfig],
):
return MultiModalKwargsItems.from_hf_inputs(hf_inputs, config_by_key) \
.get_data()
return MultiModalKwargsItems.from_hf_inputs(hf_inputs, config_by_key).get_data()
@staticmethod
@deprecated("`MultiModalKwargs.from_items` is deprecated and "
"will be removed in v0.13. "
"Please use `MultiModalKwargsItems.from_seq` and "
"access the tensor data using `.get_data()`.")
@deprecated(
"`MultiModalKwargs.from_items` is deprecated and "
"will be removed in v0.13. "
"Please use `MultiModalKwargsItems.from_seq` and "
"access the tensor data using `.get_data()`."
)
def from_items(
items: Sequence[MultiModalKwargsItem],
*,
pin_memory: bool = False,
):
return MultiModalKwargsItems.from_seq(items) \
.get_data(pin_memory=pin_memory)
return MultiModalKwargsItems.from_seq(items).get_data(pin_memory=pin_memory)
@staticmethod
def _try_stack(nested_tensors: NestedTensors,
pin_memory: bool = False) -> NestedTensors:
def _try_stack(
nested_tensors: NestedTensors, pin_memory: bool = False
) -> NestedTensors:
"""
Stack the inner dimensions that have the same shape in
a nested list of tensors.
@@ -849,9 +874,7 @@ class MultiModalKwargs(UserDict[str, NestedTensors]):
if isinstance(nested_tensors, (int, float)):
return torch.tensor(nested_tensors)
stacked = [
MultiModalKwargs._try_stack(t, pin_memory) for t in nested_tensors
]
stacked = [MultiModalKwargs._try_stack(t, pin_memory) for t in nested_tensors]
if not is_list_of(stacked, torch.Tensor, check="all"):
# Only tensors (not lists) can be stacked.
return stacked
@@ -867,16 +890,19 @@ class MultiModalKwargs(UserDict[str, NestedTensors]):
# The tensors have incompatible shapes and can't be stacked.
return tensors_
outputs = torch.empty(len(tensors_),
*tensors_[0].shape,
dtype=tensors_[0].dtype,
device=tensors_[0].device,
pin_memory=pin_memory)
outputs = torch.empty(
len(tensors_),
*tensors_[0].shape,
dtype=tensors_[0].dtype,
device=tensors_[0].device,
pin_memory=pin_memory,
)
return torch.stack(tensors_, out=outputs)
@staticmethod
def batch(inputs_list: list["MultiModalKwargs"],
pin_memory: bool = False) -> BatchedTensorInputs:
def batch(
inputs_list: list["MultiModalKwargs"], pin_memory: bool = False
) -> BatchedTensorInputs:
"""
Batch multiple inputs together into a dictionary.
@@ -915,8 +941,10 @@ class MultiModalKwargs(UserDict[str, NestedTensors]):
def __getitem__(self, key: str):
if key not in self:
raise KeyError(f"Keyword argument {key!r} not found. "
f"Available keys: {set(self.keys())}")
raise KeyError(
f"Keyword argument {key!r} not found. "
f"Available keys: {set(self.keys())}"
)
return super().__getitem__(key)
+75 -49
View File
@@ -4,8 +4,16 @@
from abc import ABC, abstractmethod
from collections import UserDict
from collections.abc import Callable, Iterator, Mapping, Sequence
from typing import (TYPE_CHECKING, Any, Generic, Literal, NamedTuple, Optional,
TypeVar, Union)
from typing import (
TYPE_CHECKING,
Any,
Generic,
Literal,
NamedTuple,
Optional,
TypeVar,
Union,
)
import numpy as np
import torch
@@ -14,9 +22,18 @@ from typing_extensions import TypeAlias, TypeGuard, assert_never
from vllm.utils import LazyLoader, is_list_of
from .audio import AudioResampler
from .inputs import (AudioItem, HfAudioItem, HfImageItem, HfVideoItem,
ImageItem, ModalityData, MultiModalDataDict,
MultiModalFieldConfig, MultiModalKwargsItems, VideoItem)
from .inputs import (
AudioItem,
HfAudioItem,
HfImageItem,
HfVideoItem,
ImageItem,
ModalityData,
MultiModalDataDict,
MultiModalFieldConfig,
MultiModalKwargsItems,
VideoItem,
)
_T = TypeVar("_T")
_I = TypeVar("_I")
@@ -40,8 +57,7 @@ class ModalityDataItems(ABC, Generic[_T, _I]):
self.modality = modality
def __repr__(self) -> str:
return (f"{type(self).__name__}(modality={self.modality!r}, "
f"len={len(self)})")
return f"{type(self).__name__}(modality={self.modality!r}, len={len(self)})"
def __len__(self) -> int:
return self.get_count()
@@ -51,8 +67,7 @@ class ModalityDataItems(ABC, Generic[_T, _I]):
if TYPE_CHECKING:
# Auto-generated
def __iter__(self) -> Iterator[_I]:
...
def __iter__(self) -> Iterator[_I]: ...
@abstractmethod
def get_count(self) -> int:
@@ -95,8 +110,9 @@ class ProcessorBatchItems(ModalityDataItems[Sequence[_T], _T]):
return {}
class EmbeddingItems(ModalityDataItems[Union[torch.Tensor, list[torch.Tensor]],
torch.Tensor]):
class EmbeddingItems(
ModalityDataItems[Union[torch.Tensor, list[torch.Tensor]], torch.Tensor]
):
"""
Base class for data items that are expressed as a batched embedding tensor,
or a list of embedding tensors (one per item).
@@ -118,8 +134,9 @@ class EmbeddingItems(ModalityDataItems[Union[torch.Tensor, list[torch.Tensor]],
return len(self.get(item_idx))
class DictEmbeddingItems(ModalityDataItems[Mapping[str, torch.Tensor],
Mapping[str, torch.Tensor]]):
class DictEmbeddingItems(
ModalityDataItems[Mapping[str, torch.Tensor], Mapping[str, torch.Tensor]]
):
"""
Base class for data items that are expressed as a dictionary of tensors.
@@ -143,8 +160,10 @@ class DictEmbeddingItems(ModalityDataItems[Mapping[str, torch.Tensor],
missing_required_data_keys = required_fields - data.keys()
if missing_required_data_keys:
data_keys = set(data.keys())
msg = (f"The data should contain the fields: {required_fields}, "
f"but only found the following keys: {data_keys}")
msg = (
f"The data should contain the fields: {required_fields}, "
f"but only found the following keys: {data_keys}"
)
raise ValueError(msg)
fields_config = fields_factory(data)
@@ -176,7 +195,6 @@ class DictEmbeddingItems(ModalityDataItems[Mapping[str, torch.Tensor],
class AudioProcessorItems(ProcessorBatchItems[HfAudioItem]):
def __init__(self, data: Optional[Sequence[HfAudioItem]]) -> None:
if data is None:
data = [None]
@@ -188,7 +206,6 @@ class AudioProcessorItems(ProcessorBatchItems[HfAudioItem]):
class AudioEmbeddingItems(EmbeddingItems):
def __init__(self, data: Union[torch.Tensor, list[torch.Tensor]]) -> None:
super().__init__(data, "audio")
@@ -199,7 +216,6 @@ class ImageSize(NamedTuple):
class ImageProcessorItems(ProcessorBatchItems[HfImageItem]):
def __init__(self, data: Optional[Sequence[HfImageItem]]) -> None:
if data is None:
data = [None]
@@ -218,18 +234,17 @@ class ImageProcessorItems(ProcessorBatchItems[HfImageItem]):
class ImageEmbeddingItems(EmbeddingItems):
def __init__(self, data: Union[torch.Tensor, list[torch.Tensor]]) -> None:
super().__init__(data, "image")
class VideoProcessorItems(ProcessorBatchItems[HfVideoItem]):
def __init__(
self,
data: Optional[Sequence[HfVideoItem]],
metadata: Optional[Union[dict[str, Any],
list[Optional[dict[str, Any]]]]] = None,
metadata: Optional[
Union[dict[str, Any], list[Optional[dict[str, Any]]]]
] = None,
) -> None:
if data is None:
data = [None]
@@ -252,7 +267,6 @@ class VideoProcessorItems(ProcessorBatchItems[HfVideoItem]):
class VideoEmbeddingItems(EmbeddingItems):
def __init__(self, data: Union[torch.Tensor, list[torch.Tensor]]) -> None:
super().__init__(data, "video")
@@ -276,8 +290,10 @@ class MultiModalDataItems(UserDict[str, ModalityDataItems[Any, Any]]):
if modality not in self:
if strict:
available_modalities = set(self.keys())
raise KeyError(f"Modality {modality!r} not found. "
f"Available modalities: {available_modalities}")
raise KeyError(
f"Modality {modality!r} not found. "
f"Available modalities: {available_modalities}"
)
return 0
@@ -298,20 +314,25 @@ class MultiModalDataItems(UserDict[str, ModalityDataItems[Any, Any]]):
"""
if modality not in self:
available_modalities = set(self.keys())
raise KeyError(f"Modality {modality!r} not found. "
f"Available modalities: {available_modalities}")
raise KeyError(
f"Modality {modality!r} not found. "
f"Available modalities: {available_modalities}"
)
items = self[modality]
if not isinstance(items, typ):
raise TypeError(f"Invalid type of data items for {modality=}. "
f"Expected type: {typ}, but "
f"found type: {type(items)}")
raise TypeError(
f"Invalid type of data items for {modality=}. "
f"Expected type: {typ}, but "
f"found type: {type(items)}"
)
return items # type: ignore[return-value]
ModalityDataParser: TypeAlias = Callable[[ModalityData[Any]],
Optional[ModalityDataItems[Any, Any]]]
ModalityDataParser: TypeAlias = Callable[
[ModalityData[Any]], Optional[ModalityDataItems[Any, Any]]
]
class MultiModalDataParser:
@@ -340,7 +361,7 @@ class MultiModalDataParser:
self.video_needs_metadata = video_needs_metadata
def _is_embeddings(
self, data: object
self, data: object
) -> TypeGuard[Union[torch.Tensor, list[torch.Tensor]]]:
if isinstance(data, torch.Tensor):
return data.ndim == 3
@@ -395,17 +416,20 @@ class MultiModalDataParser:
return AudioProcessorItems(None)
# also check single audio item with sampling rate
if self._is_empty(data) or (isinstance(data, tuple)
and self._is_empty(data[0])):
if self._is_empty(data) or (
isinstance(data, tuple) and self._is_empty(data[0])
):
return None
if self._is_embeddings(data):
return AudioEmbeddingItems(data)
if (is_list_of(data, float)
or isinstance(data,
(np.ndarray, torch.Tensor)) and data.ndim == 1
or isinstance(data, tuple)):
if (
is_list_of(data, float)
or isinstance(data, (np.ndarray, torch.Tensor))
and data.ndim == 1
or isinstance(data, tuple)
):
data_items = [data]
elif isinstance(data, (np.ndarray, torch.Tensor)):
data_items = [elem for elem in data]
@@ -418,8 +442,7 @@ class MultiModalDataParser:
if orig_sr is None:
new_audio = audio
else:
new_audio = self.audio_resampler.resample(audio,
orig_sr=orig_sr)
new_audio = self.audio_resampler.resample(audio, orig_sr=orig_sr)
new_audios.append(new_audio)
@@ -438,9 +461,11 @@ class MultiModalDataParser:
if self._is_embeddings(data):
return ImageEmbeddingItems(data)
if (isinstance(data, PILImage.Image)
or isinstance(data,
(np.ndarray, torch.Tensor)) and data.ndim == 3):
if (
isinstance(data, PILImage.Image)
or isinstance(data, (np.ndarray, torch.Tensor))
and data.ndim == 3
):
data_items = [data]
elif isinstance(data, (np.ndarray, torch.Tensor)):
data_items = [elem for elem in data]
@@ -462,9 +487,11 @@ class MultiModalDataParser:
if self._is_embeddings(data):
return VideoEmbeddingItems(data)
if (is_list_of(data, PILImage.Image)
or isinstance(data,
(np.ndarray, torch.Tensor)) and data.ndim == 4):
if (
is_list_of(data, PILImage.Image)
or isinstance(data, (np.ndarray, torch.Tensor))
and data.ndim == 4
):
data_items = [data]
elif isinstance(data, (np.ndarray, torch.Tensor)):
data_items = [elem for elem in data]
@@ -495,8 +522,7 @@ class MultiModalDataParser:
"video": self._parse_video_data,
}
def parse_mm_data(self,
mm_data: MultiModalDataDict) -> MultiModalDataItems:
def parse_mm_data(self, mm_data: MultiModalDataDict) -> MultiModalDataItems:
subparsers = self._get_subparsers()
mm_items = MultiModalDataItems()
+173 -156
View File
@@ -3,13 +3,21 @@
import time
from abc import ABC, abstractmethod
from collections import defaultdict
from collections.abc import (Callable, Generator, ItemsView, Iterable, Mapping,
Sequence)
from collections.abc import Callable, Generator, ItemsView, Iterable, Mapping, Sequence
from dataclasses import dataclass, field, replace
from enum import Enum
from functools import lru_cache
from typing import (TYPE_CHECKING, Any, Generic, NamedTuple, Optional,
Protocol, Union, cast, overload)
from typing import (
TYPE_CHECKING,
Any,
Generic,
NamedTuple,
Optional,
Protocol,
Union,
cast,
overload,
)
import regex as re
import torch
@@ -17,20 +25,28 @@ from typing_extensions import TypeVar, assert_never
from vllm.logger import init_logger
from vllm.transformers_utils.processor import cached_processor_from_config
from vllm.transformers_utils.tokenizer import (AnyTokenizer, decode_tokens,
encode_tokens)
from vllm.utils import (flatten_2d_lists, full_groupby,
get_allowed_kwarg_only_overrides)
from vllm.transformers_utils.tokenizer import AnyTokenizer, decode_tokens, encode_tokens
from vllm.utils import flatten_2d_lists, full_groupby, get_allowed_kwarg_only_overrides
from vllm.utils.jsontree import JSONTree, json_map_leaves
from .hasher import MultiModalHasher
from .inputs import (MultiModalDataDict, MultiModalEncDecInputs,
MultiModalFieldConfig, MultiModalInputs,
MultiModalKwargsItem, MultiModalKwargsItems,
MultiModalKwargsOptionalItems, MultiModalUUIDDict,
PlaceholderRange)
from .parse import (DictEmbeddingItems, EmbeddingItems, MultiModalDataItems,
MultiModalDataParser)
from .inputs import (
MultiModalDataDict,
MultiModalEncDecInputs,
MultiModalFieldConfig,
MultiModalInputs,
MultiModalKwargsItem,
MultiModalKwargsItems,
MultiModalKwargsOptionalItems,
MultiModalUUIDDict,
PlaceholderRange,
)
from .parse import (
DictEmbeddingItems,
EmbeddingItems,
MultiModalDataItems,
MultiModalDataParser,
)
if TYPE_CHECKING:
from transformers.configuration_utils import PretrainedConfig
@@ -57,9 +73,7 @@ def _cached_encode(
*,
add_special_tokens: Optional[bool] = None,
) -> list[int]:
return encode_tokens(tokenizer,
text,
add_special_tokens=add_special_tokens)
return encode_tokens(tokenizer, text, add_special_tokens=add_special_tokens)
@lru_cache(maxsize=2048)
@@ -69,9 +83,9 @@ def _cached_decode(
*,
skip_special_tokens: Optional[bool] = None,
) -> str:
return decode_tokens(tokenizer,
list(token_ids),
skip_special_tokens=skip_special_tokens)
return decode_tokens(
tokenizer, list(token_ids), skip_special_tokens=skip_special_tokens
)
def _seq2text(tokenizer: AnyTokenizer, seq: PromptSeq) -> str:
@@ -89,24 +103,22 @@ def _seq2tokens(tokenizer: AnyTokenizer, seq: PromptSeq) -> list[int]:
class _GetMatchIndex(Protocol):
def __call__(
self,
tokenizer: AnyTokenizer,
prompt: PromptSeq,
start_idx: int = 0,
) -> Optional[int]:
...
) -> Optional[int]: ...
@dataclass
class PromptIndex:
"""Resolves to an index in the prompt."""
get_match_index: _GetMatchIndex
class PromptIndexTargets:
@staticmethod
def start() -> PromptIndex:
"""
@@ -139,9 +151,7 @@ class PromptIndexTargets:
else:
if isinstance(prefix, str):
# Make both `list[int]`
prefix = encode_tokens(tokenizer,
prefix,
add_special_tokens=False)
prefix = encode_tokens(tokenizer, prefix, add_special_tokens=False)
match_idx = len(prefix)
return match_idx if prompt[:match_idx] == prefix else None
@@ -181,8 +191,7 @@ class PromptUpdateDetails(Generic[_S]):
full: _S
"""The full content."""
is_embed: Optional[Callable[[AnyTokenizer, PromptSeq],
torch.Tensor]] = None
is_embed: Optional[Callable[[AnyTokenizer, PromptSeq], torch.Tensor]] = None
"""
Given [`full`][vllm.multimodal.processing.PromptUpdateDetails.full],
return a boolean mask of shape `(len(full),)` indicating which positions
@@ -203,7 +212,6 @@ class PromptUpdateDetails(Generic[_S]):
seq: _S,
embed_text: str,
) -> "PromptUpdateDetails[_S]":
def is_embed(tokenizer: AnyTokenizer, full: PromptSeq) -> torch.Tensor:
embed_token_ids = encode_tokens(tokenizer, embed_text)
token_ids = _seq2tokens(tokenizer, full)
@@ -220,7 +228,6 @@ class PromptUpdateDetails(Generic[_S]):
seq: _S,
embed_token_id: int,
) -> "PromptUpdateDetails[_S]":
def is_embed(tokenizer: AnyTokenizer, full: PromptSeq) -> torch.Tensor:
token_ids = _seq2tokens(tokenizer, full)
@@ -238,8 +245,7 @@ use [`PromptUpdateDetails`][vllm.multimodal.processing.PromptUpdateDetails] to
specify which part.
"""
PromptUpdateContent = Union[Callable[[int], PromptUpdateInfo],
PromptUpdateInfo]
PromptUpdateContent = Union[Callable[[int], PromptUpdateInfo], PromptUpdateInfo]
"""
Given the index of the processed item within
[`modality`][vllm.multimodal.processing.PromptUpdate.modality],
@@ -408,11 +414,13 @@ class PromptReplacement(PromptUpdate):
modality="image",
target="<image>",
replacement=PromptUpdateDetails(
full="".join([
"<image_bos>",
"<image>" * image_feature_size,
"<image_eos>",
]),
full="".join(
[
"<image_bos>",
"<image>" * image_feature_size,
"<image_eos>",
]
),
features="<image>" * image_feature_size,
),
)
@@ -426,8 +434,9 @@ class PromptReplacement(PromptUpdate):
modality="image",
target=[image_token_id],
replacement=PromptUpdateDetails(
full=([image_bos_id] + [image_token_id] * image_feature_size
+ [image_eos_id]),
full=(
[image_bos_id] + [image_token_id] * image_feature_size + [image_eos_id]
),
features=[image_token_id] * image_feature_size,
),
)
@@ -459,10 +468,8 @@ class _HasModalityAttr(Protocol):
class _HasModalityProp(Protocol):
@property
def modality(self) -> str:
...
def modality(self) -> str: ...
_M = TypeVar("_M", bound=Union[_HasModalityAttr, _HasModalityProp])
@@ -520,9 +527,7 @@ class ResolvedPromptUpdate:
target_token_ids = _seq2tokens(tokenizer, target)
for match in iter_token_matches(prompt,
target_token_ids,
start_idx=start_idx):
for match in iter_token_matches(prompt, target_token_ids, start_idx=start_idx):
yield PromptTargetMatch(match.start_idx, match.end_idx)
def iter_text_matches(
@@ -544,8 +549,7 @@ class ResolvedPromptUpdate:
target_text = _seq2text(tokenizer, target)
for match in re.finditer(re.escape(target_text), prompt,
pos=start_idx):
for match in re.finditer(re.escape(target_text), prompt, pos=start_idx):
yield PromptTargetMatch(match.start(), match.end())
def iter_matches(
@@ -557,9 +561,7 @@ class ResolvedPromptUpdate:
) -> Generator[PromptTargetMatch]:
"""Yield each instance of `self.target` found in `prompt`."""
if isinstance(prompt, str):
return self.iter_text_matches(prompt,
tokenizer,
start_idx=start_idx)
return self.iter_text_matches(prompt, tokenizer, start_idx=start_idx)
return self.iter_token_matches(prompt, tokenizer, start_idx=start_idx)
@@ -680,9 +682,9 @@ def _find_matches(
break # Already found a match for this item
for match in update.iter_matches(
prompt,
tokenizer,
start_idx=prev_end_idx,
prompt,
tokenizer,
start_idx=prev_end_idx,
):
# All matches should share the same mode
if mode is None:
@@ -723,8 +725,7 @@ def _apply_matches(
out_seqs = list[Union[str, list[int]]]()
out_result: MultiModalPromptUpdatesApplyResult = {
m: [None] * len(items)
for m, items in mm_prompt_updates.items()
m: [None] * len(items) for m, items in mm_prompt_updates.items()
}
start_idx = prev_end_idx = 0
@@ -743,8 +744,7 @@ def _apply_matches(
for (modality, item_idx), (match, update_idx) in matches_to_apply:
found = True
matched_update = mm_prompt_updates[modality][item_idx][
update_idx]
matched_update = mm_prompt_updates[modality][item_idx][update_idx]
matched_content = matched_update.content.full
if mode == UpdateMode.INSERT:
@@ -756,9 +756,10 @@ def _apply_matches(
out_seqs.append(prompt[prev_end_idx:end_idx_to_insert])
out_seqs.append(
_seq2text(tokenizer, matched_content
) if isinstance(prompt, str) else _seq2tokens(
tokenizer, matched_content))
_seq2text(tokenizer, matched_content)
if isinstance(prompt, str)
else _seq2tokens(tokenizer, matched_content)
)
out_result[modality][item_idx] = update_idx
# Exclude overlapping matches
@@ -784,8 +785,7 @@ def apply_token_matches(
the same placeholder tokens. In that case, the modality that
appears earlier in `mm_prompt_updates` takes priority.
"""
token_id_seqs, result = _apply_matches(prompt, mm_prompt_updates,
tokenizer)
token_id_seqs, result = _apply_matches(prompt, mm_prompt_updates, tokenizer)
return flatten_2d_lists(token_id_seqs), result
@@ -847,8 +847,7 @@ def _iter_placeholders(
if prompt[start_idx:end_idx_full] == content_tokens_full:
content_is_embed = content.is_embed
if content_is_embed is not None:
content_is_embed = content_is_embed(
tokenizer, content.full)
content_is_embed = content_is_embed(tokenizer, content.full)
yield PlaceholderFeaturesInfo(
modality=modality,
@@ -899,16 +898,14 @@ class InputProcessingContext:
"""The tokenizer used to tokenize the inputs."""
@overload
def get_hf_config(self, /) -> "PretrainedConfig":
...
def get_hf_config(self, /) -> "PretrainedConfig": ...
@overload
def get_hf_config(
self,
typ: Union[type[_C], tuple[type[_C], ...]],
/,
) -> _C:
...
) -> _C: ...
def get_hf_config(
self,
@@ -930,9 +927,11 @@ class InputProcessingContext:
hf_config = self.model_config.hf_config
if not isinstance(hf_config, typ):
raise TypeError("Invalid type of HuggingFace config. "
f"Expected type: {typ}, but "
f"found type: {type(hf_config)}")
raise TypeError(
"Invalid type of HuggingFace config. "
f"Expected type: {typ}, but "
f"found type: {type(hf_config)}"
)
return hf_config
@@ -956,8 +955,7 @@ class InputProcessingContext:
return mm_config
@overload
def get_hf_processor(self, /, **kwargs: object) -> "ProcessorMixin":
...
def get_hf_processor(self, /, **kwargs: object) -> "ProcessorMixin": ...
@overload
def get_hf_processor(
@@ -965,8 +963,7 @@ class InputProcessingContext:
typ: Union[type[_P], tuple[type[_P], ...]],
/,
**kwargs: object,
) -> _P:
...
) -> _P: ...
def get_hf_processor(
self,
@@ -1017,7 +1014,6 @@ class InputProcessingContext:
self,
output: JSONTree,
) -> JSONTree:
def _postprocess_one(x: object):
if isinstance(x, torch.Tensor): # noqa: SIM102
# This mimics the behavior of transformers.BatchFeature
@@ -1054,17 +1050,21 @@ class InputProcessingContext:
)
try:
output = hf_processor(**data,
**allowed_kwargs,
return_tensors="pt")
output = hf_processor(**data, **allowed_kwargs, return_tensors="pt")
except Exception as exc:
# See https://github.com/huggingface/tokenizers/issues/537
if (isinstance(exc, RuntimeError) and exc
and exc.args[0] == "Already borrowed"
and num_tries < max_tries):
if (
isinstance(exc, RuntimeError)
and exc
and exc.args[0] == "Already borrowed"
and num_tries < max_tries
):
logger.warning(
"Failed to acquire tokenizer in current thread. "
"Retrying (%d/%d)...", num_tries, max_tries)
"Retrying (%d/%d)...",
num_tries,
max_tries,
)
time.sleep(0.5)
return self.call_hf_processor(
hf_processor,
@@ -1074,8 +1074,10 @@ class InputProcessingContext:
max_tries=max_tries,
)
msg = (f"Failed to apply {type(hf_processor).__name__} "
f"on data={data} with kwargs={allowed_kwargs}")
msg = (
f"Failed to apply {type(hf_processor).__name__} "
f"on data={data} with kwargs={allowed_kwargs}"
)
raise ValueError(msg) from exc
@@ -1142,8 +1144,11 @@ class BaseProcessingInfo:
for modality, supported_limit in supported_mm_limits.items():
user_limit = mm_config.get_limit_per_prompt(modality)
allowed_limits[modality] = (user_limit if supported_limit is None
else min(user_limit, supported_limit))
allowed_limits[modality] = (
user_limit
if supported_limit is None
else min(user_limit, supported_limit)
)
return allowed_limits
@@ -1154,7 +1159,7 @@ class BaseProcessingInfo:
) -> Optional[Mapping[str, int]]:
"""
Return the maximum number of tokens per item of for each modality.
When `None` (the default) is returned, vLLM will generate dummy inputs
(images/videos) at maximum possible sizes and process them to determine
the maximum token count per modality.
@@ -1165,7 +1170,7 @@ class BaseProcessingInfo:
counts, avoiding the need for dummy input generation and processing.
Note:
The maximum number of tokens per item of each modality returned
The maximum number of tokens per item of each modality returned
from this function should respect the model's maximum sequence
length and the maximum number of items of each modality allowed,
and agree with dummy inputs (images/videos) at maximum possible
@@ -1245,10 +1250,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
*,
mm_uuids: Optional[MultiModalUUIDDict] = None,
) -> MultiModalInputs:
return self.apply(prompt,
mm_data,
hf_processor_mm_kwargs,
mm_uuids=mm_uuids)
return self.apply(prompt, mm_data, hf_processor_mm_kwargs, mm_uuids=mm_uuids)
def _get_data_parser(self) -> MultiModalDataParser:
"""
@@ -1276,8 +1278,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
limit = min(supported_limit, allowed_limit)
if num_items > limit:
msg = (f"At most {limit} {modality}(s) may be provided in "
"one prompt.")
msg = f"At most {limit} {modality}(s) may be provided in one prompt."
if num_items <= supported_limit:
msg += " Set `--limit-mm-per-prompt` to increase this limit."
@@ -1339,8 +1340,10 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
mm_item_counts: Mapping[str, int],
) -> MultiModalPromptUpdates:
return {
modality: [[update.resolve(item_idx) for update in updates]
for item_idx in range(mm_item_counts.get(modality, 0))]
modality: [
[update.resolve(item_idx) for update in updates]
for item_idx in range(mm_item_counts.get(modality, 0))
]
for modality, updates in full_groupby_modality(prompt_updates)
}
@@ -1385,8 +1388,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
) -> Mapping[str, list[PlaceholderFeaturesInfo]]:
tokenizer = self.info.get_tokenizer()
return find_mm_placeholders(new_token_ids, mm_prompt_updates,
tokenizer)
return find_mm_placeholders(new_token_ids, mm_prompt_updates, tokenizer)
def _get_hf_mm_data(
self,
@@ -1436,7 +1438,8 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
"""
return not any(
isinstance(items, (EmbeddingItems, DictEmbeddingItems))
for items in mm_items.values())
for items in mm_items.values()
)
def _apply_hf_processor_text_mm(
self,
@@ -1461,7 +1464,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
)
processed_data.update(passthrough_data)
prompt_ids, = processed_data.pop("input_ids").tolist()
(prompt_ids,) = processed_data.pop("input_ids").tolist()
is_update_applied = self._hf_processor_applies_updates(
prompt_text=prompt_text,
@@ -1564,8 +1567,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
tokenization_kwargs=tokenization_kwargs,
)
prompt_ids = self._apply_hf_processor_text_only(
prompt, tokenization_kwargs)
prompt_ids = self._apply_hf_processor_text_only(prompt, tokenization_kwargs)
else:
prompt_ids = self._apply_hf_processor_tokens_only(prompt)
@@ -1611,10 +1613,11 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
# hash if `hf_processor_mm_kwargs` or `tokenization_kwargs`
# are provided. This is because the processed multimodal
# inputs can be different depending on the processor kwargs.
if item_uuid is None or \
hf_processor_mm_kwargs or \
tokenization_kwargs:
if (
item_uuid is None
or hf_processor_mm_kwargs
or tokenization_kwargs
):
# NOTE: use provided hash string to hash with kwargs
# if available for better performance.
item = item_uuid if item_uuid is not None else item
@@ -1623,16 +1626,20 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
model_id=model_id,
**{modality: item},
**hf_processor_mm_kwargs,
**tokenization_kwargs))
**tokenization_kwargs,
)
)
else:
computed.append(item_uuid)
hashes[modality] = computed
else:
hashes[modality] = [
MultiModalHasher.hash_kwargs(model_id=model_id,
**{modality: item},
**hf_processor_mm_kwargs,
**tokenization_kwargs)
MultiModalHasher.hash_kwargs(
model_id=model_id,
**{modality: item},
**hf_processor_mm_kwargs,
**tokenization_kwargs,
)
for item in items
]
@@ -1645,13 +1652,13 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
mm_hashes: MultiModalHashes,
) -> MultiModalDataItems:
mm_is_cached = {
modality: cache.is_cached(hashes)
for modality, hashes in mm_hashes.items()
modality: cache.is_cached(hashes) for modality, hashes in mm_hashes.items()
}
mm_missing_idxs = {
modality: [
idx for idx, item_is_cached in enumerate(items_is_cached)
idx
for idx, item_is_cached in enumerate(items_is_cached)
if not item_is_cached
]
for modality, items_is_cached in mm_is_cached.items()
@@ -1664,7 +1671,8 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
if data is None:
raise ValueError(
f"Cache miss for {modality} at index {idx} "
f"but data is not provided.")
f"but data is not provided."
)
else:
missing_modality_data.append(data)
mm_missing_data[modality] = missing_modality_data
@@ -1692,20 +1700,18 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
# Need to calculate this at the beginning to avoid skipping cache logic
# for subsequently repeated items in the same modality
mm_is_cached = {
modality: cache.is_cached(hashes)
for modality, hashes in mm_hashes.items()
modality: cache.is_cached(hashes) for modality, hashes in mm_hashes.items()
}
mm_missing_next_idx = defaultdict[str, int](lambda: 0)
merged_kwargs = defaultdict[str,
list[Optional[MultiModalKwargsItem]]](list)
merged_prompt_updates = defaultdict[
str, list[Sequence[ResolvedPromptUpdate]]](list)
merged_kwargs = defaultdict[str, list[Optional[MultiModalKwargsItem]]](list)
merged_prompt_updates = defaultdict[str, list[Sequence[ResolvedPromptUpdate]]](
list
)
for modality, hashes in mm_hashes.items():
missing_kwargs = mm_missing_kwargs.get(modality, [])
missing_prompt_updates = mm_missing_prompt_updates.get(
modality, [])
missing_prompt_updates = mm_missing_prompt_updates.get(modality, [])
for item_idx, item_hash in enumerate(hashes):
kwargs: Optional[MultiModalKwargsItem]
@@ -1723,10 +1729,12 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
kwargs, updates = cache.get_and_update_item(item, item_hash)
merged_kwargs[modality].append(kwargs)
merged_prompt_updates[modality].append([
self._recompute_cached_prompt_update(update, item_idx)
for update in updates
])
merged_prompt_updates[modality].append(
[
self._recompute_cached_prompt_update(update, item_idx)
for update in updates
]
)
mm_kwargs = MultiModalKwargsItems(merged_kwargs)
mm_prompt_updates = dict(merged_prompt_updates)
@@ -1756,15 +1764,16 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
mm_kwargs = MultiModalKwargsItems.from_hf_inputs(
mm_processed_data,
self._get_mm_fields_config(mm_processed_data,
hf_processor_mm_kwargs),
self._get_mm_fields_config(mm_processed_data, hf_processor_mm_kwargs),
)
# Use overrides if provided; fallback to data-dependent hashing.
mm_hashes = self._hash_mm_items(mm_data_items,
hf_processor_mm_kwargs,
tokenization_kwargs,
mm_uuids=mm_uuids)
mm_hashes = self._hash_mm_items(
mm_data_items,
hf_processor_mm_kwargs,
tokenization_kwargs,
mm_uuids=mm_uuids,
)
mm_prompt_updates = self._get_mm_prompt_updates(
mm_data_items,
@@ -1805,10 +1814,12 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
mm_uuids=mm_uuids,
)
mm_hashes = self._hash_mm_items(mm_data_items,
hf_processor_mm_kwargs,
tokenization_kwargs,
mm_uuids=mm_uuids)
mm_hashes = self._hash_mm_items(
mm_data_items,
hf_processor_mm_kwargs,
tokenization_kwargs,
mm_uuids=mm_uuids,
)
mm_missing_data_items = self._get_cache_missing_items(
cache=cache,
@@ -1833,8 +1844,9 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
mm_missing_kwargs = MultiModalKwargsItems.from_hf_inputs(
mm_missing_processed_data,
self._get_mm_fields_config(mm_missing_processed_data,
hf_processor_mm_kwargs),
self._get_mm_fields_config(
mm_missing_processed_data, hf_processor_mm_kwargs
),
)
mm_missing_prompt_updates = self._get_mm_prompt_updates(
@@ -1897,8 +1909,9 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
# of the search text in the prompt, we instead perform string-based
# updates on the decoded token IDs, then encode them back.
if not all(
all(update_idx is not None for update_idx in update_idxs)
for update_idxs in match_result.values()):
all(update_idx is not None for update_idx in update_idxs)
for update_idxs in match_result.values()
):
new_text, match_result = self._apply_text_matches(
decode_tokens(tokenizer, token_ids),
mm_prompt_updates,
@@ -1910,16 +1923,17 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
add_special_tokens=False,
)
matched_updates = defaultdict[
str, list[Sequence[ResolvedPromptUpdate]]](list)
matched_updates = defaultdict[str, list[Sequence[ResolvedPromptUpdate]]](list)
for modality, update_idxs in match_result.items():
for item_idx, update_idx in enumerate(update_idxs):
assert update_idx is not None, (
"Failed to apply prompt replacement for "
f"mm_items[{modality!r}][{item_idx}]")
f"mm_items[{modality!r}][{item_idx}]"
)
matched_updates[modality].append(
[mm_prompt_updates[modality][item_idx][update_idx]])
[mm_prompt_updates[modality][item_idx][update_idx]]
)
placeholders = self._find_mm_placeholders(
new_token_ids,
@@ -1944,7 +1958,8 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
"There is likely a problem with your "
"implementation of merged multi-modal processor for this "
"model (usually arising from an inconsistency between "
"`_call_hf_processor` and `_get_mm_fields_config`).")
"`_call_hf_processor` and `_get_mm_fields_config`)."
)
def _validate_mm_updates(
self,
@@ -1962,7 +1977,8 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
"This is likely because you forgot to include input "
"placeholder tokens (e.g., `<image>`, `<|image_pad|>`) "
"in the prompt. If the model has a chat template, make "
"sure you have applied it before calling `LLM.generate`.")
"sure you have applied it before calling `LLM.generate`."
)
def _validate_mm_placeholders(
self,
@@ -1978,7 +1994,8 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
f"corresponding to {item_count} {modality} items, but "
f"instead found {len(placeholders)} prompt placeholders! "
"Make sure the implementation of `_call_hf_processor` and "
"`_get_mm_fields_config` are consistent with each other.")
"`_get_mm_fields_config` are consistent with each other."
)
def _maybe_apply_prompt_updates(
self,
@@ -2070,7 +2087,6 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
class EncDecMultiModalProcessor(BaseMultiModalProcessor[_I]):
@abstractmethod
def create_encoder_prompt(
self,
@@ -2104,15 +2120,16 @@ class EncDecMultiModalProcessor(BaseMultiModalProcessor[_I]):
tokenizer = self.info.get_tokenizer()
decoder_prompt_raw = self.create_decoder_prompt(prompt, mm_data)
if isinstance(decoder_prompt_raw, str):
decoder_prompt_ids = encode_tokens(tokenizer,
decoder_prompt_raw,
add_special_tokens=False)
decoder_prompt_ids = encode_tokens(
tokenizer, decoder_prompt_raw, add_special_tokens=False
)
else:
decoder_prompt_ids = decoder_prompt_raw
mm_inputs = MultiModalEncDecInputs(
encoder_prompt_token_ids=encoder_inputs["prompt_token_ids"],
**encoder_inputs)
**encoder_inputs,
)
mm_inputs["prompt_token_ids"] = decoder_prompt_ids
return mm_inputs
+55 -30
View File
@@ -10,15 +10,26 @@ import numpy.typing as npt
from PIL import Image
import vllm.envs as envs
from vllm.config.multimodal import (AudioDummyOptions, BaseDummyOptions,
ImageDummyOptions, VideoDummyOptions)
from vllm.config.multimodal import (
AudioDummyOptions,
BaseDummyOptions,
ImageDummyOptions,
VideoDummyOptions,
)
from vllm.logger import init_logger
from .inputs import (MultiModalDataDict, MultiModalEncDecInputs,
MultiModalInputs, MultiModalKwargsItems,
MultiModalPlaceholderDict)
from .processing import (BaseMultiModalProcessor, BaseProcessingInfo,
EncDecMultiModalProcessor)
from .inputs import (
MultiModalDataDict,
MultiModalEncDecInputs,
MultiModalInputs,
MultiModalKwargsItems,
MultiModalPlaceholderDict,
)
from .processing import (
BaseMultiModalProcessor,
BaseProcessingInfo,
EncDecMultiModalProcessor,
)
logger = init_logger(__name__)
@@ -29,6 +40,7 @@ class ProcessorInputs:
Represents the keyword arguments to
[`vllm.multimodal.processing.BaseMultiModalProcessor.apply`][].
"""
prompt: Union[str, list[int]]
mm_data: MultiModalDataDict
hf_processor_mm_kwargs: Mapping[str, object] = field(default_factory=dict)
@@ -86,7 +98,7 @@ class BaseDummyInputsBuilder(ABC, Generic[_I]):
mm_counts: Count of items per modality
mm_options: Configurable options per modality (optional).
If None, use model defaults for backward compatibility.
If provided, models can use these to customize dummy
If provided, models can use these to customize dummy
data generation.
"""
raise NotImplementedError
@@ -113,9 +125,11 @@ class BaseDummyInputsBuilder(ABC, Generic[_I]):
tokenization_kwargs = {"truncation": False}
return ProcessorInputs(prompt=dummy_text,
mm_data=dummy_mm_data,
tokenization_kwargs=tokenization_kwargs)
return ProcessorInputs(
prompt=dummy_text,
mm_data=dummy_mm_data,
tokenization_kwargs=tokenization_kwargs,
)
def _get_dummy_audios(
self,
@@ -130,10 +144,12 @@ class BaseDummyInputsBuilder(ABC, Generic[_I]):
if overrides.length > length:
logger.warning(
"audio.length override (%d) exceeds model's "
"maximum length (%d), will be ignored", overrides.length,
length)
"maximum length (%d), will be ignored",
overrides.length,
length,
)
length = min(length, overrides.length)
audio = np.zeros((length, ))
audio = np.zeros((length,))
return [audio] * num_audios
def _get_dummy_images(
@@ -151,15 +167,19 @@ class BaseDummyInputsBuilder(ABC, Generic[_I]):
if overrides.width > width:
logger.warning(
"image.width override (%d) exceeds model's "
"maximum width (%d), will be ignored", overrides.width,
width)
"maximum width (%d), will be ignored",
overrides.width,
width,
)
width = min(width, overrides.width)
if overrides.height:
if overrides.height > height:
logger.warning(
"image.height override (%d) exceeds model's "
"maximum height (%d), will be ignored",
overrides.height, height)
overrides.height,
height,
)
height = min(height, overrides.height)
image = Image.new("RGB", (width, height), color=255)
return [image] * num_images
@@ -181,21 +201,27 @@ class BaseDummyInputsBuilder(ABC, Generic[_I]):
logger.warning(
"video.num_frames override (%d) exceeds model's "
"maximum number of frames (%d), will be ignored",
overrides.num_frames, num_frames)
overrides.num_frames,
num_frames,
)
num_frames = min(num_frames, overrides.num_frames)
if overrides.width:
if overrides.width > width:
logger.warning(
"video.width override (%d) exceeds model's "
"maximum width (%d), will be ignored", overrides.width,
width)
"maximum width (%d), will be ignored",
overrides.width,
width,
)
width = min(width, overrides.width)
if overrides.height:
if overrides.height > height:
logger.warning(
"video.height override (%d) exceeds model's "
"maximum height (%d), will be ignored",
overrides.height, height)
overrides.height,
height,
)
height = min(height, overrides.height)
video = np.full((num_frames, width, height, 3), 255)
return [video] * num_videos
@@ -236,7 +262,8 @@ class MultiModalProfiler(Generic[_I]):
factory = self.dummy_inputs
processor_inputs = factory.get_dummy_processor_inputs(
seq_len, mm_counts, mm_options)
seq_len, mm_counts, mm_options
)
return self.processor.apply(
prompt=processor_inputs.prompt,
@@ -253,9 +280,10 @@ class MultiModalProfiler(Generic[_I]):
placeholders_by_modality = mm_inputs["mm_placeholders"]
return {
modality:
sum(item.get_num_embeds() if mm_embeddings_only else item.length
for item in placeholders)
modality: sum(
item.get_num_embeds() if mm_embeddings_only else item.length
for item in placeholders
)
for modality, placeholders in placeholders_by_modality.items()
}
@@ -330,8 +358,7 @@ class MultiModalProfiler(Generic[_I]):
return max_tokens_per_item
mm_inputs = self._get_dummy_mm_inputs(seq_len, mm_counts)
return self._get_mm_num_tokens(mm_inputs,
mm_embeddings_only=mm_embeddings_only)
return self._get_mm_num_tokens(mm_inputs, mm_embeddings_only=mm_embeddings_only)
def get_mm_max_contiguous_tokens(
self,
@@ -349,6 +376,4 @@ class MultiModalProfiler(Generic[_I]):
initializing the encoder cache size.
"""
return self._get_mm_max_tokens(seq_len,
mm_counts,
mm_embeddings_only=False)
return self._get_mm_max_tokens(seq_len, mm_counts, mm_embeddings_only=False)
+36 -34
View File
@@ -8,15 +8,21 @@ import torch.nn as nn
from vllm.config.multimodal import BaseDummyOptions
from vllm.logger import init_logger
from vllm.transformers_utils.tokenizer import (AnyTokenizer,
cached_tokenizer_from_config)
from vllm.transformers_utils.tokenizer import AnyTokenizer, cached_tokenizer_from_config
from vllm.utils import ClassRegistry
from .cache import BaseMultiModalProcessorCache
from .processing import (BaseMultiModalProcessor, BaseProcessingInfo,
InputProcessingContext)
from .profiling import (BaseDummyInputsBuilder, DummyDecoderData,
DummyEncoderData, MultiModalProfiler)
from .processing import (
BaseMultiModalProcessor,
BaseProcessingInfo,
InputProcessingContext,
)
from .profiling import (
BaseDummyInputsBuilder,
DummyDecoderData,
DummyEncoderData,
MultiModalProfiler,
)
if TYPE_CHECKING:
from vllm.config import ModelConfig
@@ -38,8 +44,7 @@ class ProcessingInfoFactory(Protocol[_I_co]):
def __call__(
self,
ctx: InputProcessingContext,
) -> _I_co:
...
) -> _I_co: ...
class DummyInputsBuilderFactory(Protocol[_I]): # type: ignore[misc]
@@ -49,8 +54,7 @@ class DummyInputsBuilderFactory(Protocol[_I]): # type: ignore[misc]
instance from the context.
"""
def __call__(self, info: _I) -> BaseDummyInputsBuilder[_I]:
...
def __call__(self, info: _I) -> BaseDummyInputsBuilder[_I]: ...
class MultiModalProcessorFactory(Protocol[_I]): # type: ignore[misc]
@@ -66,8 +70,7 @@ class MultiModalProcessorFactory(Protocol[_I]): # type: ignore[misc]
dummy_inputs: BaseDummyInputsBuilder[_I],
*,
cache: Optional[BaseMultiModalProcessorCache] = None,
) -> BaseMultiModalProcessor[_I]:
...
) -> BaseMultiModalProcessor[_I]: ...
@dataclass(frozen=True)
@@ -93,8 +96,7 @@ class MultiModalRegistry:
"""
def __init__(self) -> None:
self._processor_factories = ClassRegistry[nn.Module,
_ProcessorFactories]()
self._processor_factories = ClassRegistry[nn.Module, _ProcessorFactories]()
def _extract_mm_options(
self,
@@ -112,8 +114,7 @@ class MultiModalRegistry:
mm_options = {
m: opt
for m in model_config.multimodal_config.limit_per_prompt
if (opt := model_config.multimodal_config.get_dummy_options(m)
) is not None
if (opt := model_config.multimodal_config.get_dummy_options(m)) is not None
}
return mm_options if len(mm_options) > 0 else None
@@ -121,8 +122,8 @@ class MultiModalRegistry:
def supports_multimodal_inputs(self, model_config: "ModelConfig") -> bool:
"""
Checks if the model supports multimodal inputs.
Returns True if the model is multimodal with any non-zero supported
modalities, otherwise returns False, effectively running in
Returns True if the model is multimodal with any non-zero supported
modalities, otherwise returns False, effectively running in
text-only mode.
"""
if not model_config.is_multimodal_model:
@@ -135,11 +136,13 @@ class MultiModalRegistry:
# Check if all supported modalities have limit == 0
if all(
mm_config.get_limit_per_prompt(modality) == 0
for modality in supported_modalities):
mm_config.get_limit_per_prompt(modality) == 0
for modality in supported_modalities
):
logger.info_once(
"All limits of multimodal modalities supported by the model "
"are set to 0, running in text-only mode.")
"are set to 0, running in text-only mode."
)
return False
return True
@@ -165,10 +168,7 @@ class MultiModalRegistry:
return profiler.get_mm_max_contiguous_tokens(
seq_len,
{
modality: 1
for modality, limit in mm_limits.items() if limit > 0
},
{modality: 1 for modality, limit in mm_limits.items() if limit > 0},
)
def get_max_tokens_per_item_by_nonzero_modality(
@@ -235,7 +235,9 @@ class MultiModalRegistry:
logger.warning(
"Model class %s already has a multi-modal processor "
"registered to %s. It is overwritten by the new one.",
model_cls, self)
model_cls,
self,
)
self._processor_factories[model_cls] = _ProcessorFactories(
info=info,
@@ -315,15 +317,15 @@ class MultiModalRegistry:
# count-only behavior remains unchanged.
mm_options = self._extract_mm_options(model_config)
dummy_data = profiler.get_decoder_dummy_data(seq_len, mm_counts,
mm_options)
dummy_data = profiler.get_decoder_dummy_data(seq_len, mm_counts, mm_options)
# Having more tokens is over-conservative but otherwise fine
token_ids = dummy_data.prompt_token_ids
if len(token_ids) < seq_len:
raise AssertionError(
f"Expected at least {seq_len} dummy tokens for profiling, "
f"but found {len(token_ids)} tokens instead.")
f"but found {len(token_ids)} tokens instead."
)
return dummy_data
@@ -348,8 +350,7 @@ class MultiModalRegistry:
# count-only behavior remains unchanged.
mm_options = self._extract_mm_options(model_config)
dummy_data = profiler.get_encoder_dummy_data(seq_len, mm_counts,
mm_options)
dummy_data = profiler.get_encoder_dummy_data(seq_len, mm_counts, mm_options)
# Having more tokens is over-conservative but otherwise fine
token_ids = dummy_data.prompt_token_ids
@@ -368,15 +369,16 @@ class MultiModalRegistry:
"""
if not model_config.is_encoder_decoder:
return 0
max_tokens = self.\
get_max_tokens_per_item_by_nonzero_modality(model_config)
max_tokens = self.get_max_tokens_per_item_by_nonzero_modality(model_config)
if not max_tokens:
# TODO - this function assumes encoder-decoder models are
# multimodal. This will need to change when adding support for more
# than whisper.
return 0
assert len(max_tokens) == 1, "Encoder-decoder models are expected \
assert len(max_tokens) == 1, (
"Encoder-decoder models are expected \
to implement the multimodal interface with at most one modality."
)
first_modality = next(iter(max_tokens))
return max_tokens[first_modality]
+65 -52
View File
@@ -28,8 +28,12 @@ from .video import VideoMediaIO
_M = TypeVar("_M")
if TYPE_CHECKING:
from .inputs import (BatchedTensorInputs, MultiModalKwargsItem,
MultiModalKwargsItems, MultiModalPlaceholderDict)
from .inputs import (
BatchedTensorInputs,
MultiModalKwargsItem,
MultiModalKwargsItems,
MultiModalPlaceholderDict,
)
else:
BatchedTensorInputs = Any
MultiModalKwargsItem = Any
@@ -37,12 +41,12 @@ else:
MultiModalPlaceholderDict = Any
global_thread_pool = ThreadPoolExecutor(
max_workers=envs.VLLM_MEDIA_LOADING_THREAD_COUNT)
max_workers=envs.VLLM_MEDIA_LOADING_THREAD_COUNT
)
atexit.register(global_thread_pool.shutdown)
class MediaConnector:
def __init__(
self,
media_io_kwargs: Optional[dict[str, dict[str, Any]]] = None,
@@ -53,9 +57,9 @@ class MediaConnector:
) -> None:
"""
Args:
media_io_kwargs: Additional args passed to process media
inputs, keyed by modalities. For example,
to set num_frames for video, set
media_io_kwargs: Additional args passed to process media
inputs, keyed by modalities. For example,
to set num_frames for video, set
`--media-io-kwargs '{"video":{"num_frames":40}}'`
connection: HTTP connection client to download media contents.
allowed_local_media_path: A local directory to load media files
@@ -63,8 +67,9 @@ class MediaConnector:
"""
super().__init__()
self.media_io_kwargs: dict[str, dict[
str, Any]] = media_io_kwargs if media_io_kwargs else {}
self.media_io_kwargs: dict[str, dict[str, Any]] = (
media_io_kwargs if media_io_kwargs else {}
)
self.connection = connection
if allowed_local_media_path:
@@ -73,11 +78,13 @@ class MediaConnector:
if not allowed_local_media_path_.exists():
raise ValueError(
"Invalid `--allowed-local-media-path`: The path "
f"{allowed_local_media_path_} does not exist.")
f"{allowed_local_media_path_} does not exist."
)
if not allowed_local_media_path_.is_dir():
raise ValueError(
"Invalid `--allowed-local-media-path`: The path "
f"{allowed_local_media_path_} must be a directory.")
f"{allowed_local_media_path_} must be a directory."
)
else:
allowed_local_media_path_ = None
@@ -107,24 +114,29 @@ class MediaConnector:
) -> _M: # type: ignore[type-var]
allowed_local_media_path = self.allowed_local_media_path
if allowed_local_media_path is None:
raise RuntimeError("Cannot load local files without "
"`--allowed-local-media-path`.")
raise RuntimeError(
"Cannot load local files without `--allowed-local-media-path`."
)
filepath = Path(url2pathname(url_spec.path))
if allowed_local_media_path not in filepath.resolve().parents:
raise ValueError(
f"The file path {filepath} must be a subpath "
f"of `--allowed-local-media-path` {allowed_local_media_path}.")
f"of `--allowed-local-media-path` {allowed_local_media_path}."
)
return media_io.load_file(filepath)
def _assert_url_in_allowed_media_domains(self, url_spec) -> None:
if self.allowed_media_domains and url_spec.hostname not in \
self.allowed_media_domains:
if (
self.allowed_media_domains
and url_spec.hostname not in self.allowed_media_domains
):
raise ValueError(
f"The URL must be from one of the allowed domains: "
f"{self.allowed_media_domains}. Input URL domain: "
f"{url_spec.hostname}")
f"{url_spec.hostname}"
)
def load_from_url(
self,
@@ -175,20 +187,19 @@ class MediaConnector:
timeout=fetch_timeout,
allow_redirects=envs.VLLM_MEDIA_URL_ALLOW_REDIRECTS,
)
future = loop.run_in_executor(global_thread_pool,
media_io.load_bytes, data)
future = loop.run_in_executor(global_thread_pool, media_io.load_bytes, data)
return await future
if url_spec.scheme == "data":
future = loop.run_in_executor(global_thread_pool,
self._load_data_url, url_spec,
media_io)
future = loop.run_in_executor(
global_thread_pool, self._load_data_url, url_spec, media_io
)
return await future
if url_spec.scheme == "file":
future = loop.run_in_executor(global_thread_pool,
self._load_file_url, url_spec,
media_io)
future = loop.run_in_executor(
global_thread_pool, self._load_file_url, url_spec, media_io
)
return await future
msg = "The URL must be either a HTTP, data or file URL."
raise ValueError(msg)
@@ -234,8 +245,9 @@ class MediaConnector:
By default, the image is converted into RGB format.
"""
image_io = ImageMediaIO(image_mode=image_mode,
**self.media_io_kwargs.get("image", {}))
image_io = ImageMediaIO(
image_mode=image_mode, **self.media_io_kwargs.get("image", {})
)
try:
return self.load_from_url(
@@ -258,8 +270,9 @@ class MediaConnector:
By default, the image is converted into RGB format.
"""
image_io = ImageMediaIO(image_mode=image_mode,
**self.media_io_kwargs.get("image", {}))
image_io = ImageMediaIO(
image_mode=image_mode, **self.media_io_kwargs.get("image", {})
)
try:
return await self.load_from_url_async(
@@ -280,10 +293,10 @@ class MediaConnector:
"""
Load video from an HTTP or base64 data URL.
"""
image_io = ImageMediaIO(image_mode=image_mode,
**self.media_io_kwargs.get("image", {}))
video_io = VideoMediaIO(image_io,
**self.media_io_kwargs.get("video", {}))
image_io = ImageMediaIO(
image_mode=image_mode, **self.media_io_kwargs.get("image", {})
)
video_io = VideoMediaIO(image_io, **self.media_io_kwargs.get("video", {}))
return self.load_from_url(
video_url,
@@ -302,10 +315,10 @@ class MediaConnector:
By default, the image is converted into RGB format.
"""
image_io = ImageMediaIO(image_mode=image_mode,
**self.media_io_kwargs.get("image", {}))
video_io = VideoMediaIO(image_io,
**self.media_io_kwargs.get("video", {}))
image_io = ImageMediaIO(
image_mode=image_mode, **self.media_io_kwargs.get("image", {})
)
video_io = VideoMediaIO(image_io, **self.media_io_kwargs.get("video", {}))
return await self.load_from_url_async(
video_url,
@@ -356,7 +369,8 @@ def encode_video_base64(frames: npt.NDArray) -> str:
def argsort_mm_positions(
mm_positions: MultiModalPlaceholderDict) -> list[tuple[str, int]]:
mm_positions: MultiModalPlaceholderDict,
) -> list[tuple[str, int]]:
"""
Given a `MultiModalPlaceholderDict`, output a sequence of keys to
sort the dictionary by `offset` (starting index in the input sequence)
@@ -366,9 +380,11 @@ def argsort_mm_positions(
A list of `(modality, idx)`, which can be used to access an item
by `mm_positions[modality][idx]`.
"""
flat_items = ((modality, idx, item)
for modality, items in mm_positions.items()
for idx, item in enumerate(items))
flat_items = (
(modality, idx, item)
for modality, items in mm_positions.items()
for idx, item in enumerate(items)
)
sorted_flat_items = sorted(flat_items, key=lambda x: x[2].offset)
@@ -397,7 +413,8 @@ def group_mm_kwargs_by_modality(
raise RuntimeError(
"`group_mm_kwargs_by_modality` now requires "
"`merge_by_field_config` arg, please update your model runner "
"according to https://github.com/vllm-project/vllm/pull/25676.")
"according to https://github.com/vllm-project/vllm/pull/25676."
)
from vllm.multimodal.inputs import MultiModalKwargs, MultiModalKwargsItems
@@ -409,7 +426,9 @@ def group_mm_kwargs_by_modality(
if merge_by_field_config:
mm_kwargs_group: BatchedTensorInputs = dict(
MultiModalKwargsItems.from_seq(items_lst).get_data(
pin_memory=pin_memory))
pin_memory=pin_memory
)
)
if device is not None:
mm_kwargs_group = json_map_leaves(
@@ -440,9 +459,7 @@ def fetch_audio(
audio_url: URL of the audio file to fetch.
audio_io_kwargs: Additional kwargs passed to handle audio IO.
"""
media_io_kwargs = None if not audio_io_kwargs else {
"audio": audio_io_kwargs
}
media_io_kwargs = None if not audio_io_kwargs else {"audio": audio_io_kwargs}
media_connector = MediaConnector(media_io_kwargs=media_io_kwargs)
return media_connector.fetch_audio(audio_url)
@@ -456,9 +473,7 @@ def fetch_image(
image_url: URL of the image file to fetch.
image_io_kwargs: Additional kwargs passed to handle image IO.
"""
media_io_kwargs = None if not image_io_kwargs else {
"image": image_io_kwargs
}
media_io_kwargs = None if not image_io_kwargs else {"image": image_io_kwargs}
media_connector = MediaConnector(media_io_kwargs=media_io_kwargs)
return media_connector.fetch_image(image_url)
@@ -472,8 +487,6 @@ def fetch_video(
video_url: URL of the video file to fetch.
video_io_kwargs: Additional kwargs passed to handle video IO.
"""
media_io_kwargs = None if not video_io_kwargs else {
"video": video_io_kwargs
}
media_io_kwargs = None if not video_io_kwargs else {"video": video_io_kwargs}
media_connector = MediaConnector(media_io_kwargs=media_io_kwargs)
return media_connector.fetch_video(video_url)
+40 -46
View File
@@ -21,8 +21,9 @@ from .image import ImageMediaIO
def resize_video(frames: npt.NDArray, size: tuple[int, int]) -> npt.NDArray:
num_frames, _, _, channels = frames.shape
new_height, new_width = size
resized_frames = np.empty((num_frames, new_height, new_width, channels),
dtype=frames.dtype)
resized_frames = np.empty(
(num_frames, new_height, new_width, channels), dtype=frames.dtype
)
# lazy import cv2 to avoid bothering users who only use text models
import cv2
@@ -40,8 +41,7 @@ def rescale_video_size(frames: npt.NDArray, size_factor: float) -> npt.NDArray:
return resize_video(frames, (new_height, new_width))
def sample_frames_from_video(frames: npt.NDArray,
num_frames: int) -> npt.NDArray:
def sample_frames_from_video(frames: npt.NDArray, num_frames: int) -> npt.NDArray:
total_frames = frames.shape[0]
if num_frames == -1:
return frames
@@ -52,23 +52,19 @@ def sample_frames_from_video(frames: npt.NDArray,
class VideoLoader:
@classmethod
@abstractmethod
def load_bytes(cls,
data: bytes,
num_frames: int = -1,
**kwargs) -> tuple[npt.NDArray, dict[str, Any]]:
def load_bytes(
cls, data: bytes, num_frames: int = -1, **kwargs
) -> tuple[npt.NDArray, dict[str, Any]]:
raise NotImplementedError
class VideoLoaderRegistry:
def __init__(self) -> None:
self.name2class: dict[str, type] = {}
def register(self, name: str):
def wrap(cls_to_register):
self.name2class[name] = cls_to_register
return cls_to_register
@@ -87,7 +83,6 @@ VIDEO_LOADER_REGISTRY = VideoLoaderRegistry()
@VIDEO_LOADER_REGISTRY.register("opencv")
class OpenCVVideoBackend(VideoLoader):
def get_cv2_video_api(self):
import cv2.videoio_registry as vr
@@ -127,10 +122,9 @@ class OpenCVVideoBackend(VideoLoader):
num_frames = total_frames_num
frame_idx = list(range(0, num_frames))
else:
uniform_sampled_frames = np.linspace(0,
total_frames_num - 1,
num_frames,
dtype=int)
uniform_sampled_frames = np.linspace(
0, total_frames_num - 1, num_frames, dtype=int
)
frame_idx = uniform_sampled_frames.tolist()
width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))
@@ -148,8 +142,10 @@ class OpenCVVideoBackend(VideoLoader):
frames[i] = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
i += 1
assert i == num_frames, (f"Expected reading {num_frames} frames, "
f"but only loaded {i} frames from video.")
assert i == num_frames, (
f"Expected reading {num_frames} frames, "
f"but only loaded {i} frames from video."
)
# Use transformers transformers.video_utils.VideoMetadata format
# NOTE(Isotr0py): For models like Qwen3-VL/GLM4.5V, this metadata
@@ -170,7 +166,6 @@ class OpenCVVideoBackend(VideoLoader):
@VIDEO_LOADER_REGISTRY.register("opencv_dynamic")
class OpenCVDynamicVideoBackend(OpenCVVideoBackend):
@classmethod
def load_bytes(
cls,
@@ -200,28 +195,28 @@ class OpenCVDynamicVideoBackend(OpenCVVideoBackend):
frame_indices: Union[range, list[int]]
if duration <= max_duration:
n = int(math.floor(duration * fps))
frame_indices = sorted({
min(max_frame_idx, int(math.ceil(i * original_fps / fps)))
for i in range(n)
})
frame_indices = sorted(
{
min(max_frame_idx, int(math.ceil(i * original_fps / fps)))
for i in range(n)
}
)
else:
num_samples = int(max_duration * fps)
if num_samples >= total_frames_num:
frame_indices = range(total_frames_num)
else:
target_seconds = np.linspace(0,
duration,
num_samples,
endpoint=True)
frame_indices = sorted({
min(max_frame_idx, int(math.ceil(t * original_fps)))
for t in target_seconds
})
target_seconds = np.linspace(0, duration, num_samples, endpoint=True)
frame_indices = sorted(
{
min(max_frame_idx, int(math.ceil(t * original_fps)))
for t in target_seconds
}
)
width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))
height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))
frames = np.empty((len(frame_indices), height, width, 3),
dtype=np.uint8)
frames = np.empty((len(frame_indices), height, width, 3), dtype=np.uint8)
i = 0
for idx in range(total_frames_num):
@@ -236,7 +231,8 @@ class OpenCVDynamicVideoBackend(OpenCVVideoBackend):
assert i == len(frame_indices), (
f"Expected reading {len(frame_indices)} frames, "
f"but only loaded {i} frames from video.")
f"but only loaded {i} frames from video."
)
# Use transformers transformers.video_utils.VideoMetadata format
metadata = {
@@ -252,7 +248,6 @@ class OpenCVDynamicVideoBackend(OpenCVVideoBackend):
class VideoMediaIO(MediaIO[npt.NDArray]):
def __init__(
self,
image_io: ImageMediaIO,
@@ -273,22 +268,22 @@ class VideoMediaIO(MediaIO[npt.NDArray]):
self.video_loader = VIDEO_LOADER_REGISTRY.load(video_loader_backend)
def load_bytes(self, data: bytes) -> tuple[npt.NDArray, dict[str, Any]]:
return self.video_loader.load_bytes(data,
num_frames=self.num_frames,
**self.kwargs)
return self.video_loader.load_bytes(
data, num_frames=self.num_frames, **self.kwargs
)
def load_base64(self, media_type: str,
data: str) -> tuple[npt.NDArray, dict[str, Any]]:
def load_base64(
self, media_type: str, data: str
) -> tuple[npt.NDArray, dict[str, Any]]:
if media_type.lower() == "video/jpeg":
load_frame = partial(
self.image_io.load_base64,
"image/jpeg",
)
return np.stack([
np.asarray(load_frame(frame_data))
for frame_data in data.split(",")
]), {}
return np.stack(
[np.asarray(load_frame(frame_data)) for frame_data in data.split(",")]
), {}
return self.load_bytes(base64.b64decode(data))
@@ -312,8 +307,7 @@ class VideoMediaIO(MediaIO[npt.NDArray]):
image_format=video_format,
)
return ",".join(
encode_frame(Image.fromarray(frame)) for frame in video)
return ",".join(encode_frame(Image.fromarray(frame)) for frame in video)
msg = "Only JPEG format is supported for now."
raise NotImplementedError(msg)