Convert formatting to use ruff instead of yapf + isort (#26247)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
@@ -1,10 +1,17 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from .hasher import MultiModalHasher
|
||||
from .inputs import (BatchedTensorInputs, ModalityData, MultiModalDataBuiltins,
|
||||
MultiModalDataDict, MultiModalKwargs,
|
||||
MultiModalKwargsItems, MultiModalPlaceholderDict,
|
||||
MultiModalUUIDDict, NestedTensors)
|
||||
from .inputs import (
|
||||
BatchedTensorInputs,
|
||||
ModalityData,
|
||||
MultiModalDataBuiltins,
|
||||
MultiModalDataDict,
|
||||
MultiModalKwargs,
|
||||
MultiModalKwargsItems,
|
||||
MultiModalPlaceholderDict,
|
||||
MultiModalUUIDDict,
|
||||
NestedTensors,
|
||||
)
|
||||
from .registry import MultiModalRegistry
|
||||
|
||||
MULTIMODAL_REGISTRY = MultiModalRegistry()
|
||||
|
||||
+14
-12
@@ -66,23 +66,25 @@ class AudioResampler:
|
||||
orig_sr: float,
|
||||
) -> npt.NDArray[np.floating]:
|
||||
if self.target_sr is None:
|
||||
raise RuntimeError("Audio resampling is not supported when "
|
||||
"`target_sr` is not provided")
|
||||
raise RuntimeError(
|
||||
"Audio resampling is not supported when `target_sr` is not provided"
|
||||
)
|
||||
if self.method == "librosa":
|
||||
return resample_audio_librosa(audio,
|
||||
orig_sr=orig_sr,
|
||||
target_sr=self.target_sr)
|
||||
return resample_audio_librosa(
|
||||
audio, orig_sr=orig_sr, target_sr=self.target_sr
|
||||
)
|
||||
elif self.method == "scipy":
|
||||
return resample_audio_scipy(audio,
|
||||
orig_sr=orig_sr,
|
||||
target_sr=self.target_sr)
|
||||
return resample_audio_scipy(
|
||||
audio, orig_sr=orig_sr, target_sr=self.target_sr
|
||||
)
|
||||
else:
|
||||
raise ValueError(f"Invalid resampling method: {self.method}. "
|
||||
"Supported methods are 'librosa' and 'scipy'.")
|
||||
raise ValueError(
|
||||
f"Invalid resampling method: {self.method}. "
|
||||
"Supported methods are 'librosa' and 'scipy'."
|
||||
)
|
||||
|
||||
|
||||
class AudioMediaIO(MediaIO[tuple[npt.NDArray, float]]):
|
||||
|
||||
def __init__(self, **kwargs) -> None:
|
||||
super().__init__()
|
||||
|
||||
@@ -113,4 +115,4 @@ class AudioMediaIO(MediaIO[tuple[npt.NDArray, float]]):
|
||||
soundfile.write(buffer, audio, sr, format="WAV")
|
||||
data = buffer.getvalue()
|
||||
|
||||
return base64.b64encode(data).decode('utf-8')
|
||||
return base64.b64encode(data).decode("utf-8")
|
||||
|
||||
@@ -9,7 +9,6 @@ _T = TypeVar("_T")
|
||||
|
||||
|
||||
class MediaIO(ABC, Generic[_T]):
|
||||
|
||||
@abstractmethod
|
||||
def load_bytes(self, data: bytes) -> _T:
|
||||
raise NotImplementedError
|
||||
|
||||
+52
-40
@@ -11,17 +11,24 @@ import torch
|
||||
from typing_extensions import TypeAlias, override
|
||||
|
||||
from vllm.distributed.device_communicators.shm_object_storage import (
|
||||
MsgpackSerde, SingleWriterShmObjectStorage, SingleWriterShmRingBuffer)
|
||||
MsgpackSerde,
|
||||
SingleWriterShmObjectStorage,
|
||||
SingleWriterShmRingBuffer,
|
||||
)
|
||||
from vllm.envs import VLLM_OBJECT_STORAGE_SHM_BUFFER_NAME
|
||||
from vllm.logger import init_logger
|
||||
from vllm.utils import GiB_bytes, LRUCache, MiB_bytes
|
||||
from vllm.utils.jsontree import (json_count_leaves, json_map_leaves,
|
||||
json_reduce_leaves)
|
||||
from vllm.utils.jsontree import json_count_leaves, json_map_leaves, json_reduce_leaves
|
||||
|
||||
from .inputs import (MultiModalBatchedField, MultiModalFeatureSpec,
|
||||
MultiModalFieldElem, MultiModalKwargs,
|
||||
MultiModalKwargsItem, MultiModalKwargsItems,
|
||||
NestedTensors)
|
||||
from .inputs import (
|
||||
MultiModalBatchedField,
|
||||
MultiModalFeatureSpec,
|
||||
MultiModalFieldElem,
|
||||
MultiModalKwargs,
|
||||
MultiModalKwargsItem,
|
||||
MultiModalKwargsItems,
|
||||
NestedTensors,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config import ModelConfig, VllmConfig
|
||||
@@ -90,7 +97,6 @@ _V = TypeVar("_V", bound=MultiModalCacheValue)
|
||||
|
||||
|
||||
class MultiModalCache:
|
||||
|
||||
@classmethod
|
||||
def get_leaf_size(cls, leaf: object) -> int:
|
||||
if isinstance(leaf, MultiModalProcessorCacheItem):
|
||||
@@ -99,8 +105,15 @@ class MultiModalCache:
|
||||
return leaf.item_size
|
||||
|
||||
# These are not subclasses of dict
|
||||
if isinstance(leaf, (MultiModalKwargs, MultiModalKwargsItems,
|
||||
MultiModalKwargsItem, MultiModalFieldElem)):
|
||||
if isinstance(
|
||||
leaf,
|
||||
(
|
||||
MultiModalKwargs,
|
||||
MultiModalKwargsItems,
|
||||
MultiModalKwargsItem,
|
||||
MultiModalFieldElem,
|
||||
),
|
||||
):
|
||||
return cls.get_item_size(leaf.data) # type: ignore
|
||||
|
||||
# sys.getsizeof doesn't work for tensors
|
||||
@@ -116,8 +129,9 @@ class MultiModalCache:
|
||||
*,
|
||||
debug: bool = False,
|
||||
) -> int:
|
||||
size = json_reduce_leaves(operator.add,
|
||||
json_map_leaves(cls.get_leaf_size, value))
|
||||
size = json_reduce_leaves(
|
||||
operator.add, json_map_leaves(cls.get_leaf_size, value)
|
||||
)
|
||||
|
||||
if debug:
|
||||
leaf_count = json_count_leaves(value)
|
||||
@@ -241,17 +255,19 @@ class BaseMultiModalCache(ABC, Generic[_I, _O]):
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
MultiModalProcessorCacheInItem: TypeAlias = \
|
||||
Optional[tuple[MultiModalKwargsItem, Sequence["ResolvedPromptUpdate"]]]
|
||||
MultiModalProcessorCacheInItem: TypeAlias = Optional[
|
||||
tuple[MultiModalKwargsItem, Sequence["ResolvedPromptUpdate"]]
|
||||
]
|
||||
|
||||
|
||||
MultiModalProcessorCacheOutItem: TypeAlias = \
|
||||
tuple[Optional[MultiModalKwargsItem], Sequence["ResolvedPromptUpdate"]]
|
||||
MultiModalProcessorCacheOutItem: TypeAlias = tuple[
|
||||
Optional[MultiModalKwargsItem], Sequence["ResolvedPromptUpdate"]
|
||||
]
|
||||
|
||||
|
||||
class BaseMultiModalProcessorCache(
|
||||
BaseMultiModalCache[MultiModalProcessorCacheInItem,
|
||||
MultiModalProcessorCacheOutItem]):
|
||||
BaseMultiModalCache[MultiModalProcessorCacheInItem, MultiModalProcessorCacheOutItem]
|
||||
):
|
||||
"""The required interface for caches on P0."""
|
||||
|
||||
@abstractmethod
|
||||
@@ -405,15 +421,13 @@ class ShmObjectStoreSenderCache(BaseMultiModalProcessorCache):
|
||||
create=True, # sender is the writer
|
||||
)
|
||||
self._shm_cache = SingleWriterShmObjectStorage(
|
||||
max_object_size=mm_config.mm_shm_cache_max_object_size_mb *
|
||||
MiB_bytes,
|
||||
max_object_size=mm_config.mm_shm_cache_max_object_size_mb * MiB_bytes,
|
||||
n_readers=self.world_size,
|
||||
ring_buffer=ring_buffer,
|
||||
serde_class=MsgpackSerde,
|
||||
)
|
||||
# cache (prompt_updates, modality) for P0 only
|
||||
self._p0_cache: dict[str, tuple[Sequence[ResolvedPromptUpdate],
|
||||
str]] = {}
|
||||
self._p0_cache: dict[str, tuple[Sequence[ResolvedPromptUpdate], str]] = {}
|
||||
|
||||
@override
|
||||
def is_cached_item(self, mm_hash: str) -> bool:
|
||||
@@ -425,12 +439,10 @@ class ShmObjectStoreSenderCache(BaseMultiModalProcessorCache):
|
||||
mm_item: MultiModalProcessorCacheInItem,
|
||||
mm_hash: str,
|
||||
) -> MultiModalProcessorCacheOutItem:
|
||||
|
||||
if self._shm_cache.is_cached(mm_hash):
|
||||
address, monotonic_id = self._shm_cache.get_cached(mm_hash)
|
||||
prompt_updates, modality = self._p0_cache[mm_hash]
|
||||
return self.address_as_item(address, monotonic_id,
|
||||
modality), prompt_updates
|
||||
return self.address_as_item(address, monotonic_id, modality), prompt_updates
|
||||
|
||||
assert mm_item is not None, f"Expected a cached item for {mm_hash=}"
|
||||
|
||||
@@ -440,15 +452,15 @@ class ShmObjectStoreSenderCache(BaseMultiModalProcessorCache):
|
||||
if len(self._p0_cache) >= 2 * len(self._shm_cache.key_index):
|
||||
self.remove_dangling_items()
|
||||
self._p0_cache[mm_hash] = mm_item[1], mm_item[0].modality
|
||||
address_item = self.address_as_item(address, monotonic_id,
|
||||
mm_item[0].modality)
|
||||
address_item = self.address_as_item(
|
||||
address, monotonic_id, mm_item[0].modality
|
||||
)
|
||||
return address_item, mm_item[1]
|
||||
except (ValueError, MemoryError) as e:
|
||||
# put may fail if the object is too large or
|
||||
# the cache is full.
|
||||
# In this case we log the error and keep the original mm_input.
|
||||
logger.debug("Failed to cache mm_input with hash %s: %s", mm_hash,
|
||||
e)
|
||||
logger.debug("Failed to cache mm_input with hash %s: %s", mm_hash, e)
|
||||
return mm_item
|
||||
|
||||
@override
|
||||
@@ -463,8 +475,9 @@ class ShmObjectStoreSenderCache(BaseMultiModalProcessorCache):
|
||||
for mm_hash in dangling_hashes:
|
||||
del self._p0_cache[mm_hash]
|
||||
|
||||
def address_as_item(self, address: int, monotonic_id: int,
|
||||
modality: str) -> MultiModalKwargsItem:
|
||||
def address_as_item(
|
||||
self, address: int, monotonic_id: int, modality: str
|
||||
) -> MultiModalKwargsItem:
|
||||
addr_elem = MultiModalFieldElem(
|
||||
modality=modality,
|
||||
key="address",
|
||||
@@ -494,9 +507,10 @@ def _enable_processor_cache(
|
||||
|
||||
def _enable_ipc_cache(vllm_config: "VllmConfig") -> bool:
|
||||
parallel_config = vllm_config.parallel_config
|
||||
supports_ipc_cache = ((parallel_config._api_process_count == 1
|
||||
and parallel_config.data_parallel_size == 1)
|
||||
or parallel_config.data_parallel_external_lb)
|
||||
supports_ipc_cache = (
|
||||
parallel_config._api_process_count == 1
|
||||
and parallel_config.data_parallel_size == 1
|
||||
) or parallel_config.data_parallel_external_lb
|
||||
|
||||
return supports_ipc_cache
|
||||
|
||||
@@ -542,8 +556,8 @@ def processor_only_cache_from_config(
|
||||
|
||||
|
||||
class BaseMultiModalReceiverCache(
|
||||
BaseMultiModalCache[Optional[MultiModalKwargsItem],
|
||||
MultiModalKwargsItem]):
|
||||
BaseMultiModalCache[Optional[MultiModalKwargsItem], MultiModalKwargsItem]
|
||||
):
|
||||
"""The required interface for caches on P1."""
|
||||
|
||||
def get_and_update_features(
|
||||
@@ -552,8 +566,7 @@ class BaseMultiModalReceiverCache(
|
||||
) -> list["MultiModalFeatureSpec"]:
|
||||
"""Update multimodal features with cached encoder outputs."""
|
||||
for feature in mm_features:
|
||||
feature.data = self.get_and_update_item(feature.data,
|
||||
feature.identifier)
|
||||
feature.data = self.get_and_update_item(feature.data, feature.identifier)
|
||||
return mm_features
|
||||
|
||||
|
||||
@@ -623,8 +636,7 @@ class ShmObjectStoreReceiverCache(BaseMultiModalReceiverCache):
|
||||
create=False, # Server is a reader
|
||||
)
|
||||
self._shm_cache = SingleWriterShmObjectStorage(
|
||||
max_object_size=mm_config.mm_shm_cache_max_object_size_mb *
|
||||
MiB_bytes,
|
||||
max_object_size=mm_config.mm_shm_cache_max_object_size_mb * MiB_bytes,
|
||||
n_readers=self.world_size,
|
||||
ring_buffer=ring_buffer,
|
||||
serde_class=MsgpackSerde,
|
||||
|
||||
+61
-40
@@ -13,8 +13,9 @@ import typing
|
||||
import torch
|
||||
|
||||
|
||||
def compute_retained_tokens_count(video_size_thw: torch.LongTensor,
|
||||
spatial_merge_size: int, q: float) -> int:
|
||||
def compute_retained_tokens_count(
|
||||
video_size_thw: torch.LongTensor, spatial_merge_size: int, q: float
|
||||
) -> int:
|
||||
"""
|
||||
Compute the number of retained tokens for a given video.
|
||||
Method ensures that we retain all the tokens from the first frame
|
||||
@@ -66,23 +67,21 @@ def compute_retention_mask(
|
||||
)
|
||||
|
||||
# Core EVS
|
||||
similarity = torch.nn.functional.cosine_similarity(video_embeds[1:, ...],
|
||||
video_embeds[:-1, ...],
|
||||
dim=-1)
|
||||
similarity = torch.nn.functional.cosine_similarity(
|
||||
video_embeds[1:, ...], video_embeds[:-1, ...], dim=-1
|
||||
)
|
||||
dissimilarity = 1 - similarity
|
||||
|
||||
# Always ensure we include all tokens from the first frame
|
||||
dissimilarity = torch.cat(
|
||||
[255 * torch.ones_like(video_embeds[:1, :, :, 0]), dissimilarity],
|
||||
dim=0)
|
||||
[255 * torch.ones_like(video_embeds[:1, :, :, 0]), dissimilarity], dim=0
|
||||
)
|
||||
|
||||
dissimilarity_flat = dissimilarity.view(-1)
|
||||
order = torch.argsort(dissimilarity_flat,
|
||||
dim=-1,
|
||||
descending=True,
|
||||
stable=True)
|
||||
retain_num_tokens = compute_retained_tokens_count(video_size_thw,
|
||||
spatial_merge_size, q)
|
||||
order = torch.argsort(dissimilarity_flat, dim=-1, descending=True, stable=True)
|
||||
retain_num_tokens = compute_retained_tokens_count(
|
||||
video_size_thw, spatial_merge_size, q
|
||||
)
|
||||
topk_indices = order[:retain_num_tokens]
|
||||
|
||||
retention_mask = torch.zeros_like(dissimilarity_flat, dtype=torch.bool)
|
||||
@@ -119,18 +118,34 @@ def compute_mrope_for_media(
|
||||
llm_grid_h = video_size_thw[1] // spatial_merge_size
|
||||
llm_grid_w = video_size_thw[2] // spatial_merge_size
|
||||
|
||||
t_index = ((torch.arange(llm_grid_t).view(-1, 1).expand(
|
||||
-1, llm_grid_h * llm_grid_w).mul(
|
||||
tokens_per_second * video_second_per_grid)).long().flatten())
|
||||
h_index = (torch.arange(llm_grid_h).view(1, -1,
|
||||
1).expand(llm_grid_t, -1,
|
||||
llm_grid_w).flatten())
|
||||
w_index = (torch.arange(llm_grid_w).view(1, 1, -1).expand(
|
||||
llm_grid_t, llm_grid_h, -1).flatten())
|
||||
llm_grid_w = (torch.tensor([llm_grid_w
|
||||
]).view(1, 1,
|
||||
1).expand(llm_grid_t, llm_grid_h,
|
||||
llm_grid_w).flatten())
|
||||
t_index = (
|
||||
(
|
||||
torch.arange(llm_grid_t)
|
||||
.view(-1, 1)
|
||||
.expand(-1, llm_grid_h * llm_grid_w)
|
||||
.mul(tokens_per_second * video_second_per_grid)
|
||||
)
|
||||
.long()
|
||||
.flatten()
|
||||
)
|
||||
h_index = (
|
||||
torch.arange(llm_grid_h)
|
||||
.view(1, -1, 1)
|
||||
.expand(llm_grid_t, -1, llm_grid_w)
|
||||
.flatten()
|
||||
)
|
||||
w_index = (
|
||||
torch.arange(llm_grid_w)
|
||||
.view(1, 1, -1)
|
||||
.expand(llm_grid_t, llm_grid_h, -1)
|
||||
.flatten()
|
||||
)
|
||||
llm_grid_w = (
|
||||
torch.tensor([llm_grid_w])
|
||||
.view(1, 1, 1)
|
||||
.expand(llm_grid_t, llm_grid_h, llm_grid_w)
|
||||
.flatten()
|
||||
)
|
||||
|
||||
positions = torch.stack([t_index, h_index, w_index, llm_grid_w], dim=1)
|
||||
return positions
|
||||
@@ -183,7 +198,8 @@ def recompute_mrope_positions(
|
||||
|
||||
# Tensors
|
||||
positions: torch.LongTensor = typing.cast(
|
||||
torch.LongTensor, mrope_positions.clone()) # (3, N)
|
||||
torch.LongTensor, mrope_positions.clone()
|
||||
) # (3, N)
|
||||
N = input_ids.numel()
|
||||
|
||||
image_mask = input_ids.eq(image_token_id)
|
||||
@@ -193,8 +209,7 @@ def recompute_mrope_positions(
|
||||
|
||||
# Early exit: no media in this chunk
|
||||
if len(multimodal_positions) == 0:
|
||||
delta = (int((positions.max().item() + 1) -
|
||||
N) if positions.numel() else -N)
|
||||
delta = int((positions.max().item() + 1) - N) if positions.numel() else -N
|
||||
return positions, delta
|
||||
|
||||
total_mm_tokens = torch.count_nonzero(media_mask)
|
||||
@@ -203,12 +218,12 @@ def recompute_mrope_positions(
|
||||
# Early exit: we've updated positions for all media tokens
|
||||
# (and consequently - for all remaining text tokens)
|
||||
if seen_mm_tokens == total_mm_tokens:
|
||||
delta = (int((positions.max().item() + 1) -
|
||||
N) if positions.numel() else -N)
|
||||
delta = int((positions.max().item() + 1) - N) if positions.numel() else -N
|
||||
return positions, delta
|
||||
|
||||
vision_start_indices = (input_ids == vision_start_token_id).nonzero(
|
||||
as_tuple=True)[0]
|
||||
vision_start_indices = (input_ids == vision_start_token_id).nonzero(as_tuple=True)[
|
||||
0
|
||||
]
|
||||
|
||||
for mm_pos in multimodal_positions:
|
||||
# Each mm_pos can be a complete embedding for single media
|
||||
@@ -218,8 +233,9 @@ def recompute_mrope_positions(
|
||||
# - Current prefill chunk has no vision start indexes at all
|
||||
# - Vision start token appeared in previous prefill round
|
||||
# - Regular case
|
||||
seen_vision_start_indices = vision_start_indices[vision_start_indices <
|
||||
num_computed_tokens]
|
||||
seen_vision_start_indices = vision_start_indices[
|
||||
vision_start_indices < num_computed_tokens
|
||||
]
|
||||
|
||||
if len(seen_vision_start_indices):
|
||||
# If we have encountered some vision start indexes,
|
||||
@@ -228,19 +244,23 @@ def recompute_mrope_positions(
|
||||
# | TTTTTTTTTSVVVVVVVVVV|VVVVVVTTTTTTTTTTTTTTTT|
|
||||
last_vision_start_token = seen_vision_start_indices[-1]
|
||||
seem_mm_tokens_before_last_vision_start = torch.count_nonzero(
|
||||
media_mask[:last_vision_start_token])
|
||||
media_mask[:last_vision_start_token]
|
||||
)
|
||||
in_the_middle_of_media = (
|
||||
seen_mm_tokens > seem_mm_tokens_before_last_vision_start)
|
||||
seen_mm_tokens > seem_mm_tokens_before_last_vision_start
|
||||
)
|
||||
|
||||
if in_the_middle_of_media:
|
||||
mm_embeddings_seen = (seen_mm_tokens -
|
||||
seem_mm_tokens_before_last_vision_start)
|
||||
mm_embeddings_seen = (
|
||||
seen_mm_tokens - seem_mm_tokens_before_last_vision_start
|
||||
)
|
||||
global_mm_start = last_vision_start_token
|
||||
else:
|
||||
# We have completed previous mm_embedding part and
|
||||
# ready to start a new one
|
||||
next_vision_start_token = vision_start_indices[
|
||||
vision_start_indices >= num_computed_tokens][0]
|
||||
vision_start_indices >= num_computed_tokens
|
||||
][0]
|
||||
mm_embeddings_seen = 0
|
||||
global_mm_start = next_vision_start_token
|
||||
|
||||
@@ -248,7 +268,8 @@ def recompute_mrope_positions(
|
||||
# If there were no vision start indexes so far,
|
||||
# let's find first vision start index
|
||||
next_vision_start_token = vision_start_indices[
|
||||
vision_start_indices >= num_computed_tokens][0]
|
||||
vision_start_indices >= num_computed_tokens
|
||||
][0]
|
||||
|
||||
mm_embeddings_seen = 0
|
||||
global_mm_start = next_vision_start_token
|
||||
|
||||
+25
-20
@@ -17,23 +17,23 @@ logger = init_logger(__name__)
|
||||
|
||||
|
||||
class MultiModalHasher:
|
||||
|
||||
@classmethod
|
||||
def serialize_item(cls, obj: object) -> Iterable[Union[bytes, memoryview]]:
|
||||
# Simple cases
|
||||
if isinstance(obj, (bytes, memoryview)):
|
||||
return (obj, )
|
||||
return (obj,)
|
||||
if isinstance(obj, str):
|
||||
return (obj.encode("utf-8"), )
|
||||
return (obj.encode("utf-8"),)
|
||||
if isinstance(obj, (int, float)):
|
||||
return (np.array(obj).tobytes(), )
|
||||
return (np.array(obj).tobytes(),)
|
||||
|
||||
if isinstance(obj, Image.Image):
|
||||
exif = obj.getexif()
|
||||
if Image.ExifTags.Base.ImageID in exif and isinstance(
|
||||
exif[Image.ExifTags.Base.ImageID], uuid.UUID):
|
||||
exif[Image.ExifTags.Base.ImageID], uuid.UUID
|
||||
):
|
||||
# If the image has exif ImageID tag, use that
|
||||
return (exif[Image.ExifTags.Base.ImageID].bytes, )
|
||||
return (exif[Image.ExifTags.Base.ImageID].bytes,)
|
||||
data = {"mode": obj.mode, "data": np.asarray(obj)}
|
||||
if obj.palette is not None:
|
||||
data["palette"] = obj.palette.palette
|
||||
@@ -49,30 +49,35 @@ class MultiModalHasher:
|
||||
# Workaround: View the tensor as a contiguous 1D array of bytes
|
||||
if tensor_dtype == torch.bfloat16:
|
||||
tensor_obj = tensor_obj.contiguous()
|
||||
tensor_obj = tensor_obj.view(
|
||||
(tensor_obj.numel(), )).view(torch.uint8)
|
||||
tensor_obj = tensor_obj.view((tensor_obj.numel(),)).view(torch.uint8)
|
||||
|
||||
return cls.iter_item_to_bytes(
|
||||
"tensor", {
|
||||
"tensor",
|
||||
{
|
||||
"original_dtype": str(tensor_dtype),
|
||||
"original_shape": tuple(tensor_shape),
|
||||
"data": tensor_obj.numpy(),
|
||||
})
|
||||
},
|
||||
)
|
||||
return cls.iter_item_to_bytes("tensor", tensor_obj.numpy())
|
||||
if isinstance(obj, np.ndarray):
|
||||
# If the array is non-contiguous, we need to copy it first
|
||||
arr_data = obj.view(
|
||||
np.uint8).data if obj.flags.c_contiguous else obj.tobytes()
|
||||
return cls.iter_item_to_bytes("ndarray", {
|
||||
"dtype": obj.dtype.str,
|
||||
"shape": obj.shape,
|
||||
"data": arr_data,
|
||||
})
|
||||
arr_data = (
|
||||
obj.view(np.uint8).data if obj.flags.c_contiguous else obj.tobytes()
|
||||
)
|
||||
return cls.iter_item_to_bytes(
|
||||
"ndarray",
|
||||
{
|
||||
"dtype": obj.dtype.str,
|
||||
"shape": obj.shape,
|
||||
"data": arr_data,
|
||||
},
|
||||
)
|
||||
logger.warning(
|
||||
"No serialization method found for %s. "
|
||||
"Falling back to pickle.", type(obj))
|
||||
"No serialization method found for %s. Falling back to pickle.", type(obj)
|
||||
)
|
||||
|
||||
return (pickle.dumps(obj), )
|
||||
return (pickle.dumps(obj),)
|
||||
|
||||
@classmethod
|
||||
def iter_item_to_bytes(
|
||||
|
||||
+14
-13
@@ -12,9 +12,9 @@ from PIL import Image
|
||||
from .base import MediaIO
|
||||
|
||||
|
||||
def rescale_image_size(image: Image.Image,
|
||||
size_factor: float,
|
||||
transpose: int = -1) -> Image.Image:
|
||||
def rescale_image_size(
|
||||
image: Image.Image, size_factor: float, transpose: int = -1
|
||||
) -> Image.Image:
|
||||
"""Rescale the dimensions of an image by a constant factor."""
|
||||
new_width = int(image.width * size_factor)
|
||||
new_height = int(image.height * size_factor)
|
||||
@@ -26,7 +26,7 @@ def rescale_image_size(image: Image.Image,
|
||||
|
||||
def rgba_to_rgb(
|
||||
image: Image.Image,
|
||||
background_color: Union[tuple[int, int, int], list[int]] = (255, 255, 255)
|
||||
background_color: Union[tuple[int, int, int], list[int]] = (255, 255, 255),
|
||||
) -> Image.Image:
|
||||
"""Convert an RGBA image to RGB with filled background color."""
|
||||
assert image.mode == "RGBA"
|
||||
@@ -45,7 +45,6 @@ def convert_image_mode(image: Image.Image, to_mode: str):
|
||||
|
||||
|
||||
class ImageMediaIO(MediaIO[Image.Image]):
|
||||
|
||||
def __init__(self, image_mode: str = "RGB", **kwargs) -> None:
|
||||
super().__init__()
|
||||
|
||||
@@ -59,18 +58,21 @@ class ImageMediaIO(MediaIO[Image.Image]):
|
||||
|
||||
# Extract RGBA background color from kwargs if provided
|
||||
# Default to white background for backward compatibility
|
||||
rgba_bg = kwargs.get('rgba_background_color', (255, 255, 255))
|
||||
rgba_bg = kwargs.get("rgba_background_color", (255, 255, 255))
|
||||
# Convert list to tuple for consistency
|
||||
if isinstance(rgba_bg, list):
|
||||
rgba_bg = tuple(rgba_bg)
|
||||
|
||||
# Validate rgba_background_color format
|
||||
if not (isinstance(rgba_bg, tuple) and len(rgba_bg) == 3
|
||||
and all(isinstance(c, int) and 0 <= c <= 255
|
||||
for c in rgba_bg)):
|
||||
if not (
|
||||
isinstance(rgba_bg, tuple)
|
||||
and len(rgba_bg) == 3
|
||||
and all(isinstance(c, int) and 0 <= c <= 255 for c in rgba_bg)
|
||||
):
|
||||
raise ValueError(
|
||||
"rgba_background_color must be a list or tuple of 3 integers "
|
||||
"in the range [0, 255].")
|
||||
"in the range [0, 255]."
|
||||
)
|
||||
self.rgba_background_color = rgba_bg
|
||||
|
||||
def _convert_image_mode(self, image: Image.Image) -> Image.Image:
|
||||
@@ -108,11 +110,10 @@ class ImageMediaIO(MediaIO[Image.Image]):
|
||||
image.save(buffer, image_format)
|
||||
data = buffer.getvalue()
|
||||
|
||||
return pybase64.b64encode(data).decode('utf-8')
|
||||
return pybase64.b64encode(data).decode("utf-8")
|
||||
|
||||
|
||||
class ImageEmbeddingMediaIO(MediaIO[torch.Tensor]):
|
||||
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
|
||||
@@ -127,4 +128,4 @@ class ImageEmbeddingMediaIO(MediaIO[torch.Tensor]):
|
||||
return torch.load(filepath, weights_only=True)
|
||||
|
||||
def encode_base64(self, media: torch.Tensor) -> str:
|
||||
return pybase64.b64encode(media.numpy()).decode('utf-8')
|
||||
return pybase64.b64encode(media.numpy()).decode("utf-8")
|
||||
|
||||
+106
-78
@@ -7,8 +7,7 @@ from collections.abc import Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from functools import partial
|
||||
from itertools import accumulate
|
||||
from typing import (TYPE_CHECKING, Any, Literal, Optional, TypedDict, Union,
|
||||
cast, final)
|
||||
from typing import TYPE_CHECKING, Any, Literal, Optional, TypedDict, Union, cast, final
|
||||
|
||||
import numpy as np
|
||||
from typing_extensions import NotRequired, TypeAlias, TypeVar, deprecated
|
||||
@@ -35,8 +34,9 @@ A `transformers.image_utils.ImageInput` representing a single image
|
||||
item, which can be passed to a HuggingFace `ImageProcessor`.
|
||||
"""
|
||||
|
||||
HfVideoItem: TypeAlias = Union[list["Image"], np.ndarray, "torch.Tensor",
|
||||
list[np.ndarray], list["torch.Tensor"]]
|
||||
HfVideoItem: TypeAlias = Union[
|
||||
list["Image"], np.ndarray, "torch.Tensor", list[np.ndarray], list["torch.Tensor"]
|
||||
]
|
||||
"""
|
||||
A `transformers.image_utils.VideoInput` representing a single video
|
||||
item, which can be passed to a HuggingFace `VideoProcessor`.
|
||||
@@ -58,8 +58,9 @@ which are treated as image embeddings;
|
||||
these are directly passed to the model without HF processing.
|
||||
"""
|
||||
|
||||
VideoItem: TypeAlias = Union[HfVideoItem, "torch.Tensor",
|
||||
tuple[HfVideoItem, dict[str, Any]]]
|
||||
VideoItem: TypeAlias = Union[
|
||||
HfVideoItem, "torch.Tensor", tuple[HfVideoItem, dict[str, Any]]
|
||||
]
|
||||
"""
|
||||
A `transformers.video_utils.VideoInput` representing a single video item.
|
||||
This can be passed to a HuggingFace `VideoProcessor`
|
||||
@@ -70,8 +71,7 @@ which are treated as video embeddings;
|
||||
these are directly passed to the model without HF processing.
|
||||
"""
|
||||
|
||||
AudioItem: TypeAlias = Union[HfAudioItem, tuple[np.ndarray, float],
|
||||
"torch.Tensor"]
|
||||
AudioItem: TypeAlias = Union[HfAudioItem, tuple[np.ndarray, float], "torch.Tensor"]
|
||||
"""
|
||||
Represents a single audio
|
||||
item, which can be passed to a HuggingFace `AudioProcessor`.
|
||||
@@ -177,8 +177,12 @@ class PlaceholderRange:
|
||||
return nested_tensors_equal(self.is_embed, other.is_embed)
|
||||
|
||||
|
||||
NestedTensors: TypeAlias = Union[list["NestedTensors"], list["torch.Tensor"],
|
||||
"torch.Tensor", tuple["torch.Tensor", ...]]
|
||||
NestedTensors: TypeAlias = Union[
|
||||
list["NestedTensors"],
|
||||
list["torch.Tensor"],
|
||||
"torch.Tensor",
|
||||
tuple["torch.Tensor", ...],
|
||||
]
|
||||
"""
|
||||
Uses a list instead of a tensor if the dimensions of each element do not match.
|
||||
"""
|
||||
@@ -193,11 +197,13 @@ def nested_tensors_equal(a: NestedTensors, b: NestedTensors) -> bool:
|
||||
return isinstance(a, torch.Tensor) and torch.equal(b, a)
|
||||
|
||||
if isinstance(a, list):
|
||||
return (isinstance(b, list)
|
||||
and all(nested_tensors_equal(a_, b_) for a_, b_ in zip(a, b)))
|
||||
return isinstance(b, list) and all(
|
||||
nested_tensors_equal(a_, b_) for a_, b_ in zip(a, b)
|
||||
)
|
||||
if isinstance(b, list):
|
||||
return (isinstance(a, list)
|
||||
and all(nested_tensors_equal(b_, a_) for b_, a_ in zip(b, a)))
|
||||
return isinstance(a, list) and all(
|
||||
nested_tensors_equal(b_, a_) for b_, a_ in zip(b, a)
|
||||
)
|
||||
|
||||
# Both a and b are scalars
|
||||
return a == b
|
||||
@@ -214,7 +220,7 @@ A dictionary containing nested tensors which have been batched via
|
||||
class MultiModalFeatureSpec:
|
||||
"""
|
||||
Represents a single multimodal input with its processed data and metadata.
|
||||
|
||||
|
||||
Used by the V1 engine to track multimodal data through processing and
|
||||
caching. A request containing multiple multimodal items will have one
|
||||
MultiModalFeatureSpec per item.
|
||||
@@ -280,9 +286,11 @@ class MultiModalFieldElem:
|
||||
else:
|
||||
data_equal = nested_tensors_equal(self.data, other.data)
|
||||
|
||||
return ((self.modality, self.key) == (other.modality, other.key)
|
||||
and data_equal
|
||||
and type(self.field) == type(other.field)) # noqa: E721
|
||||
return (
|
||||
(self.modality, self.key) == (other.modality, other.key)
|
||||
and data_equal
|
||||
and type(self.field) == type(other.field)
|
||||
) # noqa: E721
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
@@ -385,10 +393,12 @@ class MultiModalBatchedField(BaseMultiModalField):
|
||||
return batch[0].unsqueeze(0).contiguous()
|
||||
first_shape = batch[0].shape
|
||||
if all(elem.shape == first_shape for elem in batch):
|
||||
out = torch.empty((len(batch), *batch[0].shape),
|
||||
dtype=batch[0].dtype,
|
||||
device=batch[0].device,
|
||||
pin_memory=pin_memory)
|
||||
out = torch.empty(
|
||||
(len(batch), *batch[0].shape),
|
||||
dtype=batch[0].dtype,
|
||||
device=batch[0].device,
|
||||
pin_memory=pin_memory,
|
||||
)
|
||||
return torch.stack(batch, out=out)
|
||||
|
||||
return batch
|
||||
@@ -401,6 +411,7 @@ class MultiModalFlatField(BaseMultiModalField):
|
||||
[`MultiModalFieldConfig.flat`][vllm.multimodal.inputs.MultiModalFieldConfig.flat]
|
||||
[`MultiModalFieldConfig.flat_from_sizes`][vllm.multimodal.inputs.MultiModalFieldConfig.flat_from_sizes]
|
||||
"""
|
||||
|
||||
slices: Union[Sequence[slice], Sequence[Sequence[slice]]]
|
||||
dim: int = 0
|
||||
|
||||
@@ -412,8 +423,9 @@ class MultiModalFlatField(BaseMultiModalField):
|
||||
) -> Sequence[MultiModalFieldElem]:
|
||||
field_factory = self._field_factory(modality=modality, key=key)
|
||||
if not is_list_of(self.slices, slice, check="all"):
|
||||
assert isinstance(data, torch.Tensor), \
|
||||
assert isinstance(data, torch.Tensor), (
|
||||
"torch.Tensor is required for multiple slices"
|
||||
)
|
||||
return [field_factory(data[cast(slice, s)]) for s in self.slices]
|
||||
|
||||
def _reduce_data(
|
||||
@@ -433,17 +445,19 @@ class MultiModalFlatField(BaseMultiModalField):
|
||||
dim = self.dim + (self.dim < 0) * len(batch[0].shape)
|
||||
|
||||
def _shape_before_after(tensor: torch.Tensor):
|
||||
return tensor.shape[:dim], tensor.shape[dim + 1:]
|
||||
return tensor.shape[:dim], tensor.shape[dim + 1 :]
|
||||
|
||||
first_shape = _shape_before_after(batch[0])
|
||||
|
||||
if all(_shape_before_after(elem) == first_shape for elem in batch):
|
||||
shape_before, shape_after = first_shape
|
||||
shape_concat = sum(item.shape[dim] for item in batch)
|
||||
out = torch.empty((*shape_before, shape_concat, *shape_after),
|
||||
dtype=batch[0].dtype,
|
||||
device=batch[0].device,
|
||||
pin_memory=pin_memory)
|
||||
out = torch.empty(
|
||||
(*shape_before, shape_concat, *shape_after),
|
||||
dtype=batch[0].dtype,
|
||||
device=batch[0].device,
|
||||
pin_memory=pin_memory,
|
||||
)
|
||||
return torch.concat(batch, dim=self.dim, out=out)
|
||||
|
||||
assert self.dim == 0, "dim == 0 is required for nested list"
|
||||
@@ -456,6 +470,7 @@ class MultiModalSharedField(BaseMultiModalField):
|
||||
Info:
|
||||
[`MultiModalFieldConfig.shared`][vllm.multimodal.inputs.MultiModalFieldConfig.shared]
|
||||
"""
|
||||
|
||||
batch_size: int
|
||||
|
||||
def build_elems(
|
||||
@@ -477,7 +492,6 @@ class MultiModalSharedField(BaseMultiModalField):
|
||||
|
||||
|
||||
class MultiModalFieldConfig:
|
||||
|
||||
@staticmethod
|
||||
def batched(modality: str):
|
||||
"""
|
||||
@@ -508,9 +522,11 @@ class MultiModalFieldConfig:
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def flat(modality: str,
|
||||
slices: Union[Sequence[slice], Sequence[Sequence[slice]]],
|
||||
dim: int = 0):
|
||||
def flat(
|
||||
modality: str,
|
||||
slices: Union[Sequence[slice], Sequence[Sequence[slice]]],
|
||||
dim: int = 0,
|
||||
):
|
||||
"""
|
||||
Defines a field where an element in the batch is obtained by
|
||||
slicing along the first dimension of the underlying data.
|
||||
@@ -561,9 +577,7 @@ class MultiModalFieldConfig:
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def flat_from_sizes(modality: str,
|
||||
size_per_item: "torch.Tensor",
|
||||
dim: int = 0):
|
||||
def flat_from_sizes(modality: str, size_per_item: "torch.Tensor", dim: int = 0):
|
||||
"""
|
||||
Defines a field where an element in the batch is obtained by
|
||||
slicing along the first dimension of the underlying data.
|
||||
@@ -609,13 +623,17 @@ class MultiModalFieldConfig:
|
||||
"""
|
||||
|
||||
if size_per_item.ndim != 1:
|
||||
raise ValueError("size_per_item should be a 1-D tensor, "
|
||||
f"but found shape: {size_per_item.shape}")
|
||||
raise ValueError(
|
||||
"size_per_item should be a 1-D tensor, "
|
||||
f"but found shape: {size_per_item.shape}"
|
||||
)
|
||||
|
||||
slice_idxs = [0, *accumulate(size_per_item)]
|
||||
slices = [(slice(None, None, None), ) * dim +
|
||||
(slice(slice_idxs[i], slice_idxs[i + 1]), )
|
||||
for i in range(len(size_per_item))]
|
||||
slices = [
|
||||
(slice(None, None, None),) * dim
|
||||
+ (slice(slice_idxs[i], slice_idxs[i + 1]),)
|
||||
for i in range(len(size_per_item))
|
||||
]
|
||||
|
||||
return MultiModalFieldConfig.flat(modality, slices, dim=dim)
|
||||
|
||||
@@ -745,7 +763,8 @@ class MultiModalKwargsItems(UserDict[str, Sequence[_I]]):
|
||||
if len(set(batch_sizes.values())) > 1:
|
||||
raise ValueError(
|
||||
f"Cannot merge different batch sizes for {modality=}! "
|
||||
f"Found: {batch_sizes=}")
|
||||
f"Found: {batch_sizes=}"
|
||||
)
|
||||
|
||||
batch_size = next(iter(batch_sizes.values()))
|
||||
for item_idx in range(batch_size):
|
||||
@@ -761,8 +780,10 @@ class MultiModalKwargsItems(UserDict[str, Sequence[_I]]):
|
||||
|
||||
def __getitem__(self, modality: str) -> Sequence[_I]:
|
||||
if modality not in self:
|
||||
raise KeyError(f"Modality {modality!r} not found. "
|
||||
f"Available modalities: {set(self.keys())}")
|
||||
raise KeyError(
|
||||
f"Modality {modality!r} not found. "
|
||||
f"Available modalities: {set(self.keys())}"
|
||||
)
|
||||
|
||||
return super().__getitem__(modality) # type: ignore[return-value]
|
||||
|
||||
@@ -770,8 +791,7 @@ class MultiModalKwargsItems(UserDict[str, Sequence[_I]]):
|
||||
for modality, items in self.items():
|
||||
for i, item in enumerate(items):
|
||||
if item is None:
|
||||
raise RuntimeError(
|
||||
f"Found empty mm_items[{modality}][{i}]")
|
||||
raise RuntimeError(f"Found empty mm_items[{modality}][{i}]")
|
||||
|
||||
return self # type: ignore[return-value]
|
||||
|
||||
@@ -780,17 +800,19 @@ class MultiModalKwargsItems(UserDict[str, Sequence[_I]]):
|
||||
for modality, items in self.items():
|
||||
for i, item in enumerate(items):
|
||||
if item is None:
|
||||
raise RuntimeError("Cannot build data from empty "
|
||||
f"mm_items[{modality}][{i}]")
|
||||
raise RuntimeError(
|
||||
f"Cannot build data from empty mm_items[{modality}][{i}]"
|
||||
)
|
||||
|
||||
for key, elem in item.items():
|
||||
elems_by_key[key].append(elem)
|
||||
|
||||
return MultiModalKwargs({
|
||||
key:
|
||||
elems[0].field.reduce_data(elems, pin_memory=pin_memory)
|
||||
for key, elems in elems_by_key.items()
|
||||
})
|
||||
return MultiModalKwargs(
|
||||
{
|
||||
key: elems[0].field.reduce_data(elems, pin_memory=pin_memory)
|
||||
for key, elems in elems_by_key.items()
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
MultiModalKwargsOptionalItems: TypeAlias = Union[
|
||||
@@ -806,33 +828,36 @@ class MultiModalKwargs(UserDict[str, NestedTensors]):
|
||||
"""
|
||||
|
||||
@staticmethod
|
||||
@deprecated("`MultiModalKwargs.from_hf_inputs` is deprecated and "
|
||||
"will be removed in v0.13. "
|
||||
"Please use `MultiModalKwargsItems.from_hf_inputs` and "
|
||||
"access the tensor data using `.get_data()`.")
|
||||
@deprecated(
|
||||
"`MultiModalKwargs.from_hf_inputs` is deprecated and "
|
||||
"will be removed in v0.13. "
|
||||
"Please use `MultiModalKwargsItems.from_hf_inputs` and "
|
||||
"access the tensor data using `.get_data()`."
|
||||
)
|
||||
def from_hf_inputs(
|
||||
hf_inputs: "BatchFeature",
|
||||
config_by_key: Mapping[str, MultiModalFieldConfig],
|
||||
):
|
||||
return MultiModalKwargsItems.from_hf_inputs(hf_inputs, config_by_key) \
|
||||
.get_data()
|
||||
return MultiModalKwargsItems.from_hf_inputs(hf_inputs, config_by_key).get_data()
|
||||
|
||||
@staticmethod
|
||||
@deprecated("`MultiModalKwargs.from_items` is deprecated and "
|
||||
"will be removed in v0.13. "
|
||||
"Please use `MultiModalKwargsItems.from_seq` and "
|
||||
"access the tensor data using `.get_data()`.")
|
||||
@deprecated(
|
||||
"`MultiModalKwargs.from_items` is deprecated and "
|
||||
"will be removed in v0.13. "
|
||||
"Please use `MultiModalKwargsItems.from_seq` and "
|
||||
"access the tensor data using `.get_data()`."
|
||||
)
|
||||
def from_items(
|
||||
items: Sequence[MultiModalKwargsItem],
|
||||
*,
|
||||
pin_memory: bool = False,
|
||||
):
|
||||
return MultiModalKwargsItems.from_seq(items) \
|
||||
.get_data(pin_memory=pin_memory)
|
||||
return MultiModalKwargsItems.from_seq(items).get_data(pin_memory=pin_memory)
|
||||
|
||||
@staticmethod
|
||||
def _try_stack(nested_tensors: NestedTensors,
|
||||
pin_memory: bool = False) -> NestedTensors:
|
||||
def _try_stack(
|
||||
nested_tensors: NestedTensors, pin_memory: bool = False
|
||||
) -> NestedTensors:
|
||||
"""
|
||||
Stack the inner dimensions that have the same shape in
|
||||
a nested list of tensors.
|
||||
@@ -849,9 +874,7 @@ class MultiModalKwargs(UserDict[str, NestedTensors]):
|
||||
if isinstance(nested_tensors, (int, float)):
|
||||
return torch.tensor(nested_tensors)
|
||||
|
||||
stacked = [
|
||||
MultiModalKwargs._try_stack(t, pin_memory) for t in nested_tensors
|
||||
]
|
||||
stacked = [MultiModalKwargs._try_stack(t, pin_memory) for t in nested_tensors]
|
||||
if not is_list_of(stacked, torch.Tensor, check="all"):
|
||||
# Only tensors (not lists) can be stacked.
|
||||
return stacked
|
||||
@@ -867,16 +890,19 @@ class MultiModalKwargs(UserDict[str, NestedTensors]):
|
||||
# The tensors have incompatible shapes and can't be stacked.
|
||||
return tensors_
|
||||
|
||||
outputs = torch.empty(len(tensors_),
|
||||
*tensors_[0].shape,
|
||||
dtype=tensors_[0].dtype,
|
||||
device=tensors_[0].device,
|
||||
pin_memory=pin_memory)
|
||||
outputs = torch.empty(
|
||||
len(tensors_),
|
||||
*tensors_[0].shape,
|
||||
dtype=tensors_[0].dtype,
|
||||
device=tensors_[0].device,
|
||||
pin_memory=pin_memory,
|
||||
)
|
||||
return torch.stack(tensors_, out=outputs)
|
||||
|
||||
@staticmethod
|
||||
def batch(inputs_list: list["MultiModalKwargs"],
|
||||
pin_memory: bool = False) -> BatchedTensorInputs:
|
||||
def batch(
|
||||
inputs_list: list["MultiModalKwargs"], pin_memory: bool = False
|
||||
) -> BatchedTensorInputs:
|
||||
"""
|
||||
Batch multiple inputs together into a dictionary.
|
||||
|
||||
@@ -915,8 +941,10 @@ class MultiModalKwargs(UserDict[str, NestedTensors]):
|
||||
|
||||
def __getitem__(self, key: str):
|
||||
if key not in self:
|
||||
raise KeyError(f"Keyword argument {key!r} not found. "
|
||||
f"Available keys: {set(self.keys())}")
|
||||
raise KeyError(
|
||||
f"Keyword argument {key!r} not found. "
|
||||
f"Available keys: {set(self.keys())}"
|
||||
)
|
||||
|
||||
return super().__getitem__(key)
|
||||
|
||||
|
||||
+75
-49
@@ -4,8 +4,16 @@
|
||||
from abc import ABC, abstractmethod
|
||||
from collections import UserDict
|
||||
from collections.abc import Callable, Iterator, Mapping, Sequence
|
||||
from typing import (TYPE_CHECKING, Any, Generic, Literal, NamedTuple, Optional,
|
||||
TypeVar, Union)
|
||||
from typing import (
|
||||
TYPE_CHECKING,
|
||||
Any,
|
||||
Generic,
|
||||
Literal,
|
||||
NamedTuple,
|
||||
Optional,
|
||||
TypeVar,
|
||||
Union,
|
||||
)
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
@@ -14,9 +22,18 @@ from typing_extensions import TypeAlias, TypeGuard, assert_never
|
||||
from vllm.utils import LazyLoader, is_list_of
|
||||
|
||||
from .audio import AudioResampler
|
||||
from .inputs import (AudioItem, HfAudioItem, HfImageItem, HfVideoItem,
|
||||
ImageItem, ModalityData, MultiModalDataDict,
|
||||
MultiModalFieldConfig, MultiModalKwargsItems, VideoItem)
|
||||
from .inputs import (
|
||||
AudioItem,
|
||||
HfAudioItem,
|
||||
HfImageItem,
|
||||
HfVideoItem,
|
||||
ImageItem,
|
||||
ModalityData,
|
||||
MultiModalDataDict,
|
||||
MultiModalFieldConfig,
|
||||
MultiModalKwargsItems,
|
||||
VideoItem,
|
||||
)
|
||||
|
||||
_T = TypeVar("_T")
|
||||
_I = TypeVar("_I")
|
||||
@@ -40,8 +57,7 @@ class ModalityDataItems(ABC, Generic[_T, _I]):
|
||||
self.modality = modality
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return (f"{type(self).__name__}(modality={self.modality!r}, "
|
||||
f"len={len(self)})")
|
||||
return f"{type(self).__name__}(modality={self.modality!r}, len={len(self)})"
|
||||
|
||||
def __len__(self) -> int:
|
||||
return self.get_count()
|
||||
@@ -51,8 +67,7 @@ class ModalityDataItems(ABC, Generic[_T, _I]):
|
||||
|
||||
if TYPE_CHECKING:
|
||||
# Auto-generated
|
||||
def __iter__(self) -> Iterator[_I]:
|
||||
...
|
||||
def __iter__(self) -> Iterator[_I]: ...
|
||||
|
||||
@abstractmethod
|
||||
def get_count(self) -> int:
|
||||
@@ -95,8 +110,9 @@ class ProcessorBatchItems(ModalityDataItems[Sequence[_T], _T]):
|
||||
return {}
|
||||
|
||||
|
||||
class EmbeddingItems(ModalityDataItems[Union[torch.Tensor, list[torch.Tensor]],
|
||||
torch.Tensor]):
|
||||
class EmbeddingItems(
|
||||
ModalityDataItems[Union[torch.Tensor, list[torch.Tensor]], torch.Tensor]
|
||||
):
|
||||
"""
|
||||
Base class for data items that are expressed as a batched embedding tensor,
|
||||
or a list of embedding tensors (one per item).
|
||||
@@ -118,8 +134,9 @@ class EmbeddingItems(ModalityDataItems[Union[torch.Tensor, list[torch.Tensor]],
|
||||
return len(self.get(item_idx))
|
||||
|
||||
|
||||
class DictEmbeddingItems(ModalityDataItems[Mapping[str, torch.Tensor],
|
||||
Mapping[str, torch.Tensor]]):
|
||||
class DictEmbeddingItems(
|
||||
ModalityDataItems[Mapping[str, torch.Tensor], Mapping[str, torch.Tensor]]
|
||||
):
|
||||
"""
|
||||
Base class for data items that are expressed as a dictionary of tensors.
|
||||
|
||||
@@ -143,8 +160,10 @@ class DictEmbeddingItems(ModalityDataItems[Mapping[str, torch.Tensor],
|
||||
missing_required_data_keys = required_fields - data.keys()
|
||||
if missing_required_data_keys:
|
||||
data_keys = set(data.keys())
|
||||
msg = (f"The data should contain the fields: {required_fields}, "
|
||||
f"but only found the following keys: {data_keys}")
|
||||
msg = (
|
||||
f"The data should contain the fields: {required_fields}, "
|
||||
f"but only found the following keys: {data_keys}"
|
||||
)
|
||||
raise ValueError(msg)
|
||||
|
||||
fields_config = fields_factory(data)
|
||||
@@ -176,7 +195,6 @@ class DictEmbeddingItems(ModalityDataItems[Mapping[str, torch.Tensor],
|
||||
|
||||
|
||||
class AudioProcessorItems(ProcessorBatchItems[HfAudioItem]):
|
||||
|
||||
def __init__(self, data: Optional[Sequence[HfAudioItem]]) -> None:
|
||||
if data is None:
|
||||
data = [None]
|
||||
@@ -188,7 +206,6 @@ class AudioProcessorItems(ProcessorBatchItems[HfAudioItem]):
|
||||
|
||||
|
||||
class AudioEmbeddingItems(EmbeddingItems):
|
||||
|
||||
def __init__(self, data: Union[torch.Tensor, list[torch.Tensor]]) -> None:
|
||||
super().__init__(data, "audio")
|
||||
|
||||
@@ -199,7 +216,6 @@ class ImageSize(NamedTuple):
|
||||
|
||||
|
||||
class ImageProcessorItems(ProcessorBatchItems[HfImageItem]):
|
||||
|
||||
def __init__(self, data: Optional[Sequence[HfImageItem]]) -> None:
|
||||
if data is None:
|
||||
data = [None]
|
||||
@@ -218,18 +234,17 @@ class ImageProcessorItems(ProcessorBatchItems[HfImageItem]):
|
||||
|
||||
|
||||
class ImageEmbeddingItems(EmbeddingItems):
|
||||
|
||||
def __init__(self, data: Union[torch.Tensor, list[torch.Tensor]]) -> None:
|
||||
super().__init__(data, "image")
|
||||
|
||||
|
||||
class VideoProcessorItems(ProcessorBatchItems[HfVideoItem]):
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
data: Optional[Sequence[HfVideoItem]],
|
||||
metadata: Optional[Union[dict[str, Any],
|
||||
list[Optional[dict[str, Any]]]]] = None,
|
||||
metadata: Optional[
|
||||
Union[dict[str, Any], list[Optional[dict[str, Any]]]]
|
||||
] = None,
|
||||
) -> None:
|
||||
if data is None:
|
||||
data = [None]
|
||||
@@ -252,7 +267,6 @@ class VideoProcessorItems(ProcessorBatchItems[HfVideoItem]):
|
||||
|
||||
|
||||
class VideoEmbeddingItems(EmbeddingItems):
|
||||
|
||||
def __init__(self, data: Union[torch.Tensor, list[torch.Tensor]]) -> None:
|
||||
super().__init__(data, "video")
|
||||
|
||||
@@ -276,8 +290,10 @@ class MultiModalDataItems(UserDict[str, ModalityDataItems[Any, Any]]):
|
||||
if modality not in self:
|
||||
if strict:
|
||||
available_modalities = set(self.keys())
|
||||
raise KeyError(f"Modality {modality!r} not found. "
|
||||
f"Available modalities: {available_modalities}")
|
||||
raise KeyError(
|
||||
f"Modality {modality!r} not found. "
|
||||
f"Available modalities: {available_modalities}"
|
||||
)
|
||||
|
||||
return 0
|
||||
|
||||
@@ -298,20 +314,25 @@ class MultiModalDataItems(UserDict[str, ModalityDataItems[Any, Any]]):
|
||||
"""
|
||||
if modality not in self:
|
||||
available_modalities = set(self.keys())
|
||||
raise KeyError(f"Modality {modality!r} not found. "
|
||||
f"Available modalities: {available_modalities}")
|
||||
raise KeyError(
|
||||
f"Modality {modality!r} not found. "
|
||||
f"Available modalities: {available_modalities}"
|
||||
)
|
||||
|
||||
items = self[modality]
|
||||
if not isinstance(items, typ):
|
||||
raise TypeError(f"Invalid type of data items for {modality=}. "
|
||||
f"Expected type: {typ}, but "
|
||||
f"found type: {type(items)}")
|
||||
raise TypeError(
|
||||
f"Invalid type of data items for {modality=}. "
|
||||
f"Expected type: {typ}, but "
|
||||
f"found type: {type(items)}"
|
||||
)
|
||||
|
||||
return items # type: ignore[return-value]
|
||||
|
||||
|
||||
ModalityDataParser: TypeAlias = Callable[[ModalityData[Any]],
|
||||
Optional[ModalityDataItems[Any, Any]]]
|
||||
ModalityDataParser: TypeAlias = Callable[
|
||||
[ModalityData[Any]], Optional[ModalityDataItems[Any, Any]]
|
||||
]
|
||||
|
||||
|
||||
class MultiModalDataParser:
|
||||
@@ -340,7 +361,7 @@ class MultiModalDataParser:
|
||||
self.video_needs_metadata = video_needs_metadata
|
||||
|
||||
def _is_embeddings(
|
||||
self, data: object
|
||||
self, data: object
|
||||
) -> TypeGuard[Union[torch.Tensor, list[torch.Tensor]]]:
|
||||
if isinstance(data, torch.Tensor):
|
||||
return data.ndim == 3
|
||||
@@ -395,17 +416,20 @@ class MultiModalDataParser:
|
||||
return AudioProcessorItems(None)
|
||||
|
||||
# also check single audio item with sampling rate
|
||||
if self._is_empty(data) or (isinstance(data, tuple)
|
||||
and self._is_empty(data[0])):
|
||||
if self._is_empty(data) or (
|
||||
isinstance(data, tuple) and self._is_empty(data[0])
|
||||
):
|
||||
return None
|
||||
|
||||
if self._is_embeddings(data):
|
||||
return AudioEmbeddingItems(data)
|
||||
|
||||
if (is_list_of(data, float)
|
||||
or isinstance(data,
|
||||
(np.ndarray, torch.Tensor)) and data.ndim == 1
|
||||
or isinstance(data, tuple)):
|
||||
if (
|
||||
is_list_of(data, float)
|
||||
or isinstance(data, (np.ndarray, torch.Tensor))
|
||||
and data.ndim == 1
|
||||
or isinstance(data, tuple)
|
||||
):
|
||||
data_items = [data]
|
||||
elif isinstance(data, (np.ndarray, torch.Tensor)):
|
||||
data_items = [elem for elem in data]
|
||||
@@ -418,8 +442,7 @@ class MultiModalDataParser:
|
||||
if orig_sr is None:
|
||||
new_audio = audio
|
||||
else:
|
||||
new_audio = self.audio_resampler.resample(audio,
|
||||
orig_sr=orig_sr)
|
||||
new_audio = self.audio_resampler.resample(audio, orig_sr=orig_sr)
|
||||
|
||||
new_audios.append(new_audio)
|
||||
|
||||
@@ -438,9 +461,11 @@ class MultiModalDataParser:
|
||||
if self._is_embeddings(data):
|
||||
return ImageEmbeddingItems(data)
|
||||
|
||||
if (isinstance(data, PILImage.Image)
|
||||
or isinstance(data,
|
||||
(np.ndarray, torch.Tensor)) and data.ndim == 3):
|
||||
if (
|
||||
isinstance(data, PILImage.Image)
|
||||
or isinstance(data, (np.ndarray, torch.Tensor))
|
||||
and data.ndim == 3
|
||||
):
|
||||
data_items = [data]
|
||||
elif isinstance(data, (np.ndarray, torch.Tensor)):
|
||||
data_items = [elem for elem in data]
|
||||
@@ -462,9 +487,11 @@ class MultiModalDataParser:
|
||||
if self._is_embeddings(data):
|
||||
return VideoEmbeddingItems(data)
|
||||
|
||||
if (is_list_of(data, PILImage.Image)
|
||||
or isinstance(data,
|
||||
(np.ndarray, torch.Tensor)) and data.ndim == 4):
|
||||
if (
|
||||
is_list_of(data, PILImage.Image)
|
||||
or isinstance(data, (np.ndarray, torch.Tensor))
|
||||
and data.ndim == 4
|
||||
):
|
||||
data_items = [data]
|
||||
elif isinstance(data, (np.ndarray, torch.Tensor)):
|
||||
data_items = [elem for elem in data]
|
||||
@@ -495,8 +522,7 @@ class MultiModalDataParser:
|
||||
"video": self._parse_video_data,
|
||||
}
|
||||
|
||||
def parse_mm_data(self,
|
||||
mm_data: MultiModalDataDict) -> MultiModalDataItems:
|
||||
def parse_mm_data(self, mm_data: MultiModalDataDict) -> MultiModalDataItems:
|
||||
subparsers = self._get_subparsers()
|
||||
|
||||
mm_items = MultiModalDataItems()
|
||||
|
||||
+173
-156
@@ -3,13 +3,21 @@
|
||||
import time
|
||||
from abc import ABC, abstractmethod
|
||||
from collections import defaultdict
|
||||
from collections.abc import (Callable, Generator, ItemsView, Iterable, Mapping,
|
||||
Sequence)
|
||||
from collections.abc import Callable, Generator, ItemsView, Iterable, Mapping, Sequence
|
||||
from dataclasses import dataclass, field, replace
|
||||
from enum import Enum
|
||||
from functools import lru_cache
|
||||
from typing import (TYPE_CHECKING, Any, Generic, NamedTuple, Optional,
|
||||
Protocol, Union, cast, overload)
|
||||
from typing import (
|
||||
TYPE_CHECKING,
|
||||
Any,
|
||||
Generic,
|
||||
NamedTuple,
|
||||
Optional,
|
||||
Protocol,
|
||||
Union,
|
||||
cast,
|
||||
overload,
|
||||
)
|
||||
|
||||
import regex as re
|
||||
import torch
|
||||
@@ -17,20 +25,28 @@ from typing_extensions import TypeVar, assert_never
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.transformers_utils.processor import cached_processor_from_config
|
||||
from vllm.transformers_utils.tokenizer import (AnyTokenizer, decode_tokens,
|
||||
encode_tokens)
|
||||
from vllm.utils import (flatten_2d_lists, full_groupby,
|
||||
get_allowed_kwarg_only_overrides)
|
||||
from vllm.transformers_utils.tokenizer import AnyTokenizer, decode_tokens, encode_tokens
|
||||
from vllm.utils import flatten_2d_lists, full_groupby, get_allowed_kwarg_only_overrides
|
||||
from vllm.utils.jsontree import JSONTree, json_map_leaves
|
||||
|
||||
from .hasher import MultiModalHasher
|
||||
from .inputs import (MultiModalDataDict, MultiModalEncDecInputs,
|
||||
MultiModalFieldConfig, MultiModalInputs,
|
||||
MultiModalKwargsItem, MultiModalKwargsItems,
|
||||
MultiModalKwargsOptionalItems, MultiModalUUIDDict,
|
||||
PlaceholderRange)
|
||||
from .parse import (DictEmbeddingItems, EmbeddingItems, MultiModalDataItems,
|
||||
MultiModalDataParser)
|
||||
from .inputs import (
|
||||
MultiModalDataDict,
|
||||
MultiModalEncDecInputs,
|
||||
MultiModalFieldConfig,
|
||||
MultiModalInputs,
|
||||
MultiModalKwargsItem,
|
||||
MultiModalKwargsItems,
|
||||
MultiModalKwargsOptionalItems,
|
||||
MultiModalUUIDDict,
|
||||
PlaceholderRange,
|
||||
)
|
||||
from .parse import (
|
||||
DictEmbeddingItems,
|
||||
EmbeddingItems,
|
||||
MultiModalDataItems,
|
||||
MultiModalDataParser,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from transformers.configuration_utils import PretrainedConfig
|
||||
@@ -57,9 +73,7 @@ def _cached_encode(
|
||||
*,
|
||||
add_special_tokens: Optional[bool] = None,
|
||||
) -> list[int]:
|
||||
return encode_tokens(tokenizer,
|
||||
text,
|
||||
add_special_tokens=add_special_tokens)
|
||||
return encode_tokens(tokenizer, text, add_special_tokens=add_special_tokens)
|
||||
|
||||
|
||||
@lru_cache(maxsize=2048)
|
||||
@@ -69,9 +83,9 @@ def _cached_decode(
|
||||
*,
|
||||
skip_special_tokens: Optional[bool] = None,
|
||||
) -> str:
|
||||
return decode_tokens(tokenizer,
|
||||
list(token_ids),
|
||||
skip_special_tokens=skip_special_tokens)
|
||||
return decode_tokens(
|
||||
tokenizer, list(token_ids), skip_special_tokens=skip_special_tokens
|
||||
)
|
||||
|
||||
|
||||
def _seq2text(tokenizer: AnyTokenizer, seq: PromptSeq) -> str:
|
||||
@@ -89,24 +103,22 @@ def _seq2tokens(tokenizer: AnyTokenizer, seq: PromptSeq) -> list[int]:
|
||||
|
||||
|
||||
class _GetMatchIndex(Protocol):
|
||||
|
||||
def __call__(
|
||||
self,
|
||||
tokenizer: AnyTokenizer,
|
||||
prompt: PromptSeq,
|
||||
start_idx: int = 0,
|
||||
) -> Optional[int]:
|
||||
...
|
||||
) -> Optional[int]: ...
|
||||
|
||||
|
||||
@dataclass
|
||||
class PromptIndex:
|
||||
"""Resolves to an index in the prompt."""
|
||||
|
||||
get_match_index: _GetMatchIndex
|
||||
|
||||
|
||||
class PromptIndexTargets:
|
||||
|
||||
@staticmethod
|
||||
def start() -> PromptIndex:
|
||||
"""
|
||||
@@ -139,9 +151,7 @@ class PromptIndexTargets:
|
||||
else:
|
||||
if isinstance(prefix, str):
|
||||
# Make both `list[int]`
|
||||
prefix = encode_tokens(tokenizer,
|
||||
prefix,
|
||||
add_special_tokens=False)
|
||||
prefix = encode_tokens(tokenizer, prefix, add_special_tokens=False)
|
||||
|
||||
match_idx = len(prefix)
|
||||
return match_idx if prompt[:match_idx] == prefix else None
|
||||
@@ -181,8 +191,7 @@ class PromptUpdateDetails(Generic[_S]):
|
||||
full: _S
|
||||
"""The full content."""
|
||||
|
||||
is_embed: Optional[Callable[[AnyTokenizer, PromptSeq],
|
||||
torch.Tensor]] = None
|
||||
is_embed: Optional[Callable[[AnyTokenizer, PromptSeq], torch.Tensor]] = None
|
||||
"""
|
||||
Given [`full`][vllm.multimodal.processing.PromptUpdateDetails.full],
|
||||
return a boolean mask of shape `(len(full),)` indicating which positions
|
||||
@@ -203,7 +212,6 @@ class PromptUpdateDetails(Generic[_S]):
|
||||
seq: _S,
|
||||
embed_text: str,
|
||||
) -> "PromptUpdateDetails[_S]":
|
||||
|
||||
def is_embed(tokenizer: AnyTokenizer, full: PromptSeq) -> torch.Tensor:
|
||||
embed_token_ids = encode_tokens(tokenizer, embed_text)
|
||||
token_ids = _seq2tokens(tokenizer, full)
|
||||
@@ -220,7 +228,6 @@ class PromptUpdateDetails(Generic[_S]):
|
||||
seq: _S,
|
||||
embed_token_id: int,
|
||||
) -> "PromptUpdateDetails[_S]":
|
||||
|
||||
def is_embed(tokenizer: AnyTokenizer, full: PromptSeq) -> torch.Tensor:
|
||||
token_ids = _seq2tokens(tokenizer, full)
|
||||
|
||||
@@ -238,8 +245,7 @@ use [`PromptUpdateDetails`][vllm.multimodal.processing.PromptUpdateDetails] to
|
||||
specify which part.
|
||||
"""
|
||||
|
||||
PromptUpdateContent = Union[Callable[[int], PromptUpdateInfo],
|
||||
PromptUpdateInfo]
|
||||
PromptUpdateContent = Union[Callable[[int], PromptUpdateInfo], PromptUpdateInfo]
|
||||
"""
|
||||
Given the index of the processed item within
|
||||
[`modality`][vllm.multimodal.processing.PromptUpdate.modality],
|
||||
@@ -408,11 +414,13 @@ class PromptReplacement(PromptUpdate):
|
||||
modality="image",
|
||||
target="<image>",
|
||||
replacement=PromptUpdateDetails(
|
||||
full="".join([
|
||||
"<image_bos>",
|
||||
"<image>" * image_feature_size,
|
||||
"<image_eos>",
|
||||
]),
|
||||
full="".join(
|
||||
[
|
||||
"<image_bos>",
|
||||
"<image>" * image_feature_size,
|
||||
"<image_eos>",
|
||||
]
|
||||
),
|
||||
features="<image>" * image_feature_size,
|
||||
),
|
||||
)
|
||||
@@ -426,8 +434,9 @@ class PromptReplacement(PromptUpdate):
|
||||
modality="image",
|
||||
target=[image_token_id],
|
||||
replacement=PromptUpdateDetails(
|
||||
full=([image_bos_id] + [image_token_id] * image_feature_size
|
||||
+ [image_eos_id]),
|
||||
full=(
|
||||
[image_bos_id] + [image_token_id] * image_feature_size + [image_eos_id]
|
||||
),
|
||||
features=[image_token_id] * image_feature_size,
|
||||
),
|
||||
)
|
||||
@@ -459,10 +468,8 @@ class _HasModalityAttr(Protocol):
|
||||
|
||||
|
||||
class _HasModalityProp(Protocol):
|
||||
|
||||
@property
|
||||
def modality(self) -> str:
|
||||
...
|
||||
def modality(self) -> str: ...
|
||||
|
||||
|
||||
_M = TypeVar("_M", bound=Union[_HasModalityAttr, _HasModalityProp])
|
||||
@@ -520,9 +527,7 @@ class ResolvedPromptUpdate:
|
||||
|
||||
target_token_ids = _seq2tokens(tokenizer, target)
|
||||
|
||||
for match in iter_token_matches(prompt,
|
||||
target_token_ids,
|
||||
start_idx=start_idx):
|
||||
for match in iter_token_matches(prompt, target_token_ids, start_idx=start_idx):
|
||||
yield PromptTargetMatch(match.start_idx, match.end_idx)
|
||||
|
||||
def iter_text_matches(
|
||||
@@ -544,8 +549,7 @@ class ResolvedPromptUpdate:
|
||||
|
||||
target_text = _seq2text(tokenizer, target)
|
||||
|
||||
for match in re.finditer(re.escape(target_text), prompt,
|
||||
pos=start_idx):
|
||||
for match in re.finditer(re.escape(target_text), prompt, pos=start_idx):
|
||||
yield PromptTargetMatch(match.start(), match.end())
|
||||
|
||||
def iter_matches(
|
||||
@@ -557,9 +561,7 @@ class ResolvedPromptUpdate:
|
||||
) -> Generator[PromptTargetMatch]:
|
||||
"""Yield each instance of `self.target` found in `prompt`."""
|
||||
if isinstance(prompt, str):
|
||||
return self.iter_text_matches(prompt,
|
||||
tokenizer,
|
||||
start_idx=start_idx)
|
||||
return self.iter_text_matches(prompt, tokenizer, start_idx=start_idx)
|
||||
|
||||
return self.iter_token_matches(prompt, tokenizer, start_idx=start_idx)
|
||||
|
||||
@@ -680,9 +682,9 @@ def _find_matches(
|
||||
break # Already found a match for this item
|
||||
|
||||
for match in update.iter_matches(
|
||||
prompt,
|
||||
tokenizer,
|
||||
start_idx=prev_end_idx,
|
||||
prompt,
|
||||
tokenizer,
|
||||
start_idx=prev_end_idx,
|
||||
):
|
||||
# All matches should share the same mode
|
||||
if mode is None:
|
||||
@@ -723,8 +725,7 @@ def _apply_matches(
|
||||
|
||||
out_seqs = list[Union[str, list[int]]]()
|
||||
out_result: MultiModalPromptUpdatesApplyResult = {
|
||||
m: [None] * len(items)
|
||||
for m, items in mm_prompt_updates.items()
|
||||
m: [None] * len(items) for m, items in mm_prompt_updates.items()
|
||||
}
|
||||
|
||||
start_idx = prev_end_idx = 0
|
||||
@@ -743,8 +744,7 @@ def _apply_matches(
|
||||
for (modality, item_idx), (match, update_idx) in matches_to_apply:
|
||||
found = True
|
||||
|
||||
matched_update = mm_prompt_updates[modality][item_idx][
|
||||
update_idx]
|
||||
matched_update = mm_prompt_updates[modality][item_idx][update_idx]
|
||||
matched_content = matched_update.content.full
|
||||
|
||||
if mode == UpdateMode.INSERT:
|
||||
@@ -756,9 +756,10 @@ def _apply_matches(
|
||||
|
||||
out_seqs.append(prompt[prev_end_idx:end_idx_to_insert])
|
||||
out_seqs.append(
|
||||
_seq2text(tokenizer, matched_content
|
||||
) if isinstance(prompt, str) else _seq2tokens(
|
||||
tokenizer, matched_content))
|
||||
_seq2text(tokenizer, matched_content)
|
||||
if isinstance(prompt, str)
|
||||
else _seq2tokens(tokenizer, matched_content)
|
||||
)
|
||||
out_result[modality][item_idx] = update_idx
|
||||
|
||||
# Exclude overlapping matches
|
||||
@@ -784,8 +785,7 @@ def apply_token_matches(
|
||||
the same placeholder tokens. In that case, the modality that
|
||||
appears earlier in `mm_prompt_updates` takes priority.
|
||||
"""
|
||||
token_id_seqs, result = _apply_matches(prompt, mm_prompt_updates,
|
||||
tokenizer)
|
||||
token_id_seqs, result = _apply_matches(prompt, mm_prompt_updates, tokenizer)
|
||||
|
||||
return flatten_2d_lists(token_id_seqs), result
|
||||
|
||||
@@ -847,8 +847,7 @@ def _iter_placeholders(
|
||||
if prompt[start_idx:end_idx_full] == content_tokens_full:
|
||||
content_is_embed = content.is_embed
|
||||
if content_is_embed is not None:
|
||||
content_is_embed = content_is_embed(
|
||||
tokenizer, content.full)
|
||||
content_is_embed = content_is_embed(tokenizer, content.full)
|
||||
|
||||
yield PlaceholderFeaturesInfo(
|
||||
modality=modality,
|
||||
@@ -899,16 +898,14 @@ class InputProcessingContext:
|
||||
"""The tokenizer used to tokenize the inputs."""
|
||||
|
||||
@overload
|
||||
def get_hf_config(self, /) -> "PretrainedConfig":
|
||||
...
|
||||
def get_hf_config(self, /) -> "PretrainedConfig": ...
|
||||
|
||||
@overload
|
||||
def get_hf_config(
|
||||
self,
|
||||
typ: Union[type[_C], tuple[type[_C], ...]],
|
||||
/,
|
||||
) -> _C:
|
||||
...
|
||||
) -> _C: ...
|
||||
|
||||
def get_hf_config(
|
||||
self,
|
||||
@@ -930,9 +927,11 @@ class InputProcessingContext:
|
||||
|
||||
hf_config = self.model_config.hf_config
|
||||
if not isinstance(hf_config, typ):
|
||||
raise TypeError("Invalid type of HuggingFace config. "
|
||||
f"Expected type: {typ}, but "
|
||||
f"found type: {type(hf_config)}")
|
||||
raise TypeError(
|
||||
"Invalid type of HuggingFace config. "
|
||||
f"Expected type: {typ}, but "
|
||||
f"found type: {type(hf_config)}"
|
||||
)
|
||||
|
||||
return hf_config
|
||||
|
||||
@@ -956,8 +955,7 @@ class InputProcessingContext:
|
||||
return mm_config
|
||||
|
||||
@overload
|
||||
def get_hf_processor(self, /, **kwargs: object) -> "ProcessorMixin":
|
||||
...
|
||||
def get_hf_processor(self, /, **kwargs: object) -> "ProcessorMixin": ...
|
||||
|
||||
@overload
|
||||
def get_hf_processor(
|
||||
@@ -965,8 +963,7 @@ class InputProcessingContext:
|
||||
typ: Union[type[_P], tuple[type[_P], ...]],
|
||||
/,
|
||||
**kwargs: object,
|
||||
) -> _P:
|
||||
...
|
||||
) -> _P: ...
|
||||
|
||||
def get_hf_processor(
|
||||
self,
|
||||
@@ -1017,7 +1014,6 @@ class InputProcessingContext:
|
||||
self,
|
||||
output: JSONTree,
|
||||
) -> JSONTree:
|
||||
|
||||
def _postprocess_one(x: object):
|
||||
if isinstance(x, torch.Tensor): # noqa: SIM102
|
||||
# This mimics the behavior of transformers.BatchFeature
|
||||
@@ -1054,17 +1050,21 @@ class InputProcessingContext:
|
||||
)
|
||||
|
||||
try:
|
||||
output = hf_processor(**data,
|
||||
**allowed_kwargs,
|
||||
return_tensors="pt")
|
||||
output = hf_processor(**data, **allowed_kwargs, return_tensors="pt")
|
||||
except Exception as exc:
|
||||
# See https://github.com/huggingface/tokenizers/issues/537
|
||||
if (isinstance(exc, RuntimeError) and exc
|
||||
and exc.args[0] == "Already borrowed"
|
||||
and num_tries < max_tries):
|
||||
if (
|
||||
isinstance(exc, RuntimeError)
|
||||
and exc
|
||||
and exc.args[0] == "Already borrowed"
|
||||
and num_tries < max_tries
|
||||
):
|
||||
logger.warning(
|
||||
"Failed to acquire tokenizer in current thread. "
|
||||
"Retrying (%d/%d)...", num_tries, max_tries)
|
||||
"Retrying (%d/%d)...",
|
||||
num_tries,
|
||||
max_tries,
|
||||
)
|
||||
time.sleep(0.5)
|
||||
return self.call_hf_processor(
|
||||
hf_processor,
|
||||
@@ -1074,8 +1074,10 @@ class InputProcessingContext:
|
||||
max_tries=max_tries,
|
||||
)
|
||||
|
||||
msg = (f"Failed to apply {type(hf_processor).__name__} "
|
||||
f"on data={data} with kwargs={allowed_kwargs}")
|
||||
msg = (
|
||||
f"Failed to apply {type(hf_processor).__name__} "
|
||||
f"on data={data} with kwargs={allowed_kwargs}"
|
||||
)
|
||||
|
||||
raise ValueError(msg) from exc
|
||||
|
||||
@@ -1142,8 +1144,11 @@ class BaseProcessingInfo:
|
||||
for modality, supported_limit in supported_mm_limits.items():
|
||||
user_limit = mm_config.get_limit_per_prompt(modality)
|
||||
|
||||
allowed_limits[modality] = (user_limit if supported_limit is None
|
||||
else min(user_limit, supported_limit))
|
||||
allowed_limits[modality] = (
|
||||
user_limit
|
||||
if supported_limit is None
|
||||
else min(user_limit, supported_limit)
|
||||
)
|
||||
|
||||
return allowed_limits
|
||||
|
||||
@@ -1154,7 +1159,7 @@ class BaseProcessingInfo:
|
||||
) -> Optional[Mapping[str, int]]:
|
||||
"""
|
||||
Return the maximum number of tokens per item of for each modality.
|
||||
|
||||
|
||||
When `None` (the default) is returned, vLLM will generate dummy inputs
|
||||
(images/videos) at maximum possible sizes and process them to determine
|
||||
the maximum token count per modality.
|
||||
@@ -1165,7 +1170,7 @@ class BaseProcessingInfo:
|
||||
counts, avoiding the need for dummy input generation and processing.
|
||||
|
||||
Note:
|
||||
The maximum number of tokens per item of each modality returned
|
||||
The maximum number of tokens per item of each modality returned
|
||||
from this function should respect the model's maximum sequence
|
||||
length and the maximum number of items of each modality allowed,
|
||||
and agree with dummy inputs (images/videos) at maximum possible
|
||||
@@ -1245,10 +1250,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
*,
|
||||
mm_uuids: Optional[MultiModalUUIDDict] = None,
|
||||
) -> MultiModalInputs:
|
||||
return self.apply(prompt,
|
||||
mm_data,
|
||||
hf_processor_mm_kwargs,
|
||||
mm_uuids=mm_uuids)
|
||||
return self.apply(prompt, mm_data, hf_processor_mm_kwargs, mm_uuids=mm_uuids)
|
||||
|
||||
def _get_data_parser(self) -> MultiModalDataParser:
|
||||
"""
|
||||
@@ -1276,8 +1278,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
limit = min(supported_limit, allowed_limit)
|
||||
|
||||
if num_items > limit:
|
||||
msg = (f"At most {limit} {modality}(s) may be provided in "
|
||||
"one prompt.")
|
||||
msg = f"At most {limit} {modality}(s) may be provided in one prompt."
|
||||
|
||||
if num_items <= supported_limit:
|
||||
msg += " Set `--limit-mm-per-prompt` to increase this limit."
|
||||
@@ -1339,8 +1340,10 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
mm_item_counts: Mapping[str, int],
|
||||
) -> MultiModalPromptUpdates:
|
||||
return {
|
||||
modality: [[update.resolve(item_idx) for update in updates]
|
||||
for item_idx in range(mm_item_counts.get(modality, 0))]
|
||||
modality: [
|
||||
[update.resolve(item_idx) for update in updates]
|
||||
for item_idx in range(mm_item_counts.get(modality, 0))
|
||||
]
|
||||
for modality, updates in full_groupby_modality(prompt_updates)
|
||||
}
|
||||
|
||||
@@ -1385,8 +1388,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
) -> Mapping[str, list[PlaceholderFeaturesInfo]]:
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
|
||||
return find_mm_placeholders(new_token_ids, mm_prompt_updates,
|
||||
tokenizer)
|
||||
return find_mm_placeholders(new_token_ids, mm_prompt_updates, tokenizer)
|
||||
|
||||
def _get_hf_mm_data(
|
||||
self,
|
||||
@@ -1436,7 +1438,8 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
"""
|
||||
return not any(
|
||||
isinstance(items, (EmbeddingItems, DictEmbeddingItems))
|
||||
for items in mm_items.values())
|
||||
for items in mm_items.values()
|
||||
)
|
||||
|
||||
def _apply_hf_processor_text_mm(
|
||||
self,
|
||||
@@ -1461,7 +1464,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
)
|
||||
processed_data.update(passthrough_data)
|
||||
|
||||
prompt_ids, = processed_data.pop("input_ids").tolist()
|
||||
(prompt_ids,) = processed_data.pop("input_ids").tolist()
|
||||
|
||||
is_update_applied = self._hf_processor_applies_updates(
|
||||
prompt_text=prompt_text,
|
||||
@@ -1564,8 +1567,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
tokenization_kwargs=tokenization_kwargs,
|
||||
)
|
||||
|
||||
prompt_ids = self._apply_hf_processor_text_only(
|
||||
prompt, tokenization_kwargs)
|
||||
prompt_ids = self._apply_hf_processor_text_only(prompt, tokenization_kwargs)
|
||||
else:
|
||||
prompt_ids = self._apply_hf_processor_tokens_only(prompt)
|
||||
|
||||
@@ -1611,10 +1613,11 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
# hash if `hf_processor_mm_kwargs` or `tokenization_kwargs`
|
||||
# are provided. This is because the processed multimodal
|
||||
# inputs can be different depending on the processor kwargs.
|
||||
if item_uuid is None or \
|
||||
hf_processor_mm_kwargs or \
|
||||
tokenization_kwargs:
|
||||
|
||||
if (
|
||||
item_uuid is None
|
||||
or hf_processor_mm_kwargs
|
||||
or tokenization_kwargs
|
||||
):
|
||||
# NOTE: use provided hash string to hash with kwargs
|
||||
# if available for better performance.
|
||||
item = item_uuid if item_uuid is not None else item
|
||||
@@ -1623,16 +1626,20 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
model_id=model_id,
|
||||
**{modality: item},
|
||||
**hf_processor_mm_kwargs,
|
||||
**tokenization_kwargs))
|
||||
**tokenization_kwargs,
|
||||
)
|
||||
)
|
||||
else:
|
||||
computed.append(item_uuid)
|
||||
hashes[modality] = computed
|
||||
else:
|
||||
hashes[modality] = [
|
||||
MultiModalHasher.hash_kwargs(model_id=model_id,
|
||||
**{modality: item},
|
||||
**hf_processor_mm_kwargs,
|
||||
**tokenization_kwargs)
|
||||
MultiModalHasher.hash_kwargs(
|
||||
model_id=model_id,
|
||||
**{modality: item},
|
||||
**hf_processor_mm_kwargs,
|
||||
**tokenization_kwargs,
|
||||
)
|
||||
for item in items
|
||||
]
|
||||
|
||||
@@ -1645,13 +1652,13 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
mm_hashes: MultiModalHashes,
|
||||
) -> MultiModalDataItems:
|
||||
mm_is_cached = {
|
||||
modality: cache.is_cached(hashes)
|
||||
for modality, hashes in mm_hashes.items()
|
||||
modality: cache.is_cached(hashes) for modality, hashes in mm_hashes.items()
|
||||
}
|
||||
|
||||
mm_missing_idxs = {
|
||||
modality: [
|
||||
idx for idx, item_is_cached in enumerate(items_is_cached)
|
||||
idx
|
||||
for idx, item_is_cached in enumerate(items_is_cached)
|
||||
if not item_is_cached
|
||||
]
|
||||
for modality, items_is_cached in mm_is_cached.items()
|
||||
@@ -1664,7 +1671,8 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
if data is None:
|
||||
raise ValueError(
|
||||
f"Cache miss for {modality} at index {idx} "
|
||||
f"but data is not provided.")
|
||||
f"but data is not provided."
|
||||
)
|
||||
else:
|
||||
missing_modality_data.append(data)
|
||||
mm_missing_data[modality] = missing_modality_data
|
||||
@@ -1692,20 +1700,18 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
# Need to calculate this at the beginning to avoid skipping cache logic
|
||||
# for subsequently repeated items in the same modality
|
||||
mm_is_cached = {
|
||||
modality: cache.is_cached(hashes)
|
||||
for modality, hashes in mm_hashes.items()
|
||||
modality: cache.is_cached(hashes) for modality, hashes in mm_hashes.items()
|
||||
}
|
||||
|
||||
mm_missing_next_idx = defaultdict[str, int](lambda: 0)
|
||||
|
||||
merged_kwargs = defaultdict[str,
|
||||
list[Optional[MultiModalKwargsItem]]](list)
|
||||
merged_prompt_updates = defaultdict[
|
||||
str, list[Sequence[ResolvedPromptUpdate]]](list)
|
||||
merged_kwargs = defaultdict[str, list[Optional[MultiModalKwargsItem]]](list)
|
||||
merged_prompt_updates = defaultdict[str, list[Sequence[ResolvedPromptUpdate]]](
|
||||
list
|
||||
)
|
||||
for modality, hashes in mm_hashes.items():
|
||||
missing_kwargs = mm_missing_kwargs.get(modality, [])
|
||||
missing_prompt_updates = mm_missing_prompt_updates.get(
|
||||
modality, [])
|
||||
missing_prompt_updates = mm_missing_prompt_updates.get(modality, [])
|
||||
|
||||
for item_idx, item_hash in enumerate(hashes):
|
||||
kwargs: Optional[MultiModalKwargsItem]
|
||||
@@ -1723,10 +1729,12 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
kwargs, updates = cache.get_and_update_item(item, item_hash)
|
||||
|
||||
merged_kwargs[modality].append(kwargs)
|
||||
merged_prompt_updates[modality].append([
|
||||
self._recompute_cached_prompt_update(update, item_idx)
|
||||
for update in updates
|
||||
])
|
||||
merged_prompt_updates[modality].append(
|
||||
[
|
||||
self._recompute_cached_prompt_update(update, item_idx)
|
||||
for update in updates
|
||||
]
|
||||
)
|
||||
|
||||
mm_kwargs = MultiModalKwargsItems(merged_kwargs)
|
||||
mm_prompt_updates = dict(merged_prompt_updates)
|
||||
@@ -1756,15 +1764,16 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
|
||||
mm_kwargs = MultiModalKwargsItems.from_hf_inputs(
|
||||
mm_processed_data,
|
||||
self._get_mm_fields_config(mm_processed_data,
|
||||
hf_processor_mm_kwargs),
|
||||
self._get_mm_fields_config(mm_processed_data, hf_processor_mm_kwargs),
|
||||
)
|
||||
|
||||
# Use overrides if provided; fallback to data-dependent hashing.
|
||||
mm_hashes = self._hash_mm_items(mm_data_items,
|
||||
hf_processor_mm_kwargs,
|
||||
tokenization_kwargs,
|
||||
mm_uuids=mm_uuids)
|
||||
mm_hashes = self._hash_mm_items(
|
||||
mm_data_items,
|
||||
hf_processor_mm_kwargs,
|
||||
tokenization_kwargs,
|
||||
mm_uuids=mm_uuids,
|
||||
)
|
||||
|
||||
mm_prompt_updates = self._get_mm_prompt_updates(
|
||||
mm_data_items,
|
||||
@@ -1805,10 +1814,12 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
mm_uuids=mm_uuids,
|
||||
)
|
||||
|
||||
mm_hashes = self._hash_mm_items(mm_data_items,
|
||||
hf_processor_mm_kwargs,
|
||||
tokenization_kwargs,
|
||||
mm_uuids=mm_uuids)
|
||||
mm_hashes = self._hash_mm_items(
|
||||
mm_data_items,
|
||||
hf_processor_mm_kwargs,
|
||||
tokenization_kwargs,
|
||||
mm_uuids=mm_uuids,
|
||||
)
|
||||
|
||||
mm_missing_data_items = self._get_cache_missing_items(
|
||||
cache=cache,
|
||||
@@ -1833,8 +1844,9 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
|
||||
mm_missing_kwargs = MultiModalKwargsItems.from_hf_inputs(
|
||||
mm_missing_processed_data,
|
||||
self._get_mm_fields_config(mm_missing_processed_data,
|
||||
hf_processor_mm_kwargs),
|
||||
self._get_mm_fields_config(
|
||||
mm_missing_processed_data, hf_processor_mm_kwargs
|
||||
),
|
||||
)
|
||||
|
||||
mm_missing_prompt_updates = self._get_mm_prompt_updates(
|
||||
@@ -1897,8 +1909,9 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
# of the search text in the prompt, we instead perform string-based
|
||||
# updates on the decoded token IDs, then encode them back.
|
||||
if not all(
|
||||
all(update_idx is not None for update_idx in update_idxs)
|
||||
for update_idxs in match_result.values()):
|
||||
all(update_idx is not None for update_idx in update_idxs)
|
||||
for update_idxs in match_result.values()
|
||||
):
|
||||
new_text, match_result = self._apply_text_matches(
|
||||
decode_tokens(tokenizer, token_ids),
|
||||
mm_prompt_updates,
|
||||
@@ -1910,16 +1923,17 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
add_special_tokens=False,
|
||||
)
|
||||
|
||||
matched_updates = defaultdict[
|
||||
str, list[Sequence[ResolvedPromptUpdate]]](list)
|
||||
matched_updates = defaultdict[str, list[Sequence[ResolvedPromptUpdate]]](list)
|
||||
for modality, update_idxs in match_result.items():
|
||||
for item_idx, update_idx in enumerate(update_idxs):
|
||||
assert update_idx is not None, (
|
||||
"Failed to apply prompt replacement for "
|
||||
f"mm_items[{modality!r}][{item_idx}]")
|
||||
f"mm_items[{modality!r}][{item_idx}]"
|
||||
)
|
||||
|
||||
matched_updates[modality].append(
|
||||
[mm_prompt_updates[modality][item_idx][update_idx]])
|
||||
[mm_prompt_updates[modality][item_idx][update_idx]]
|
||||
)
|
||||
|
||||
placeholders = self._find_mm_placeholders(
|
||||
new_token_ids,
|
||||
@@ -1944,7 +1958,8 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
"There is likely a problem with your "
|
||||
"implementation of merged multi-modal processor for this "
|
||||
"model (usually arising from an inconsistency between "
|
||||
"`_call_hf_processor` and `_get_mm_fields_config`).")
|
||||
"`_call_hf_processor` and `_get_mm_fields_config`)."
|
||||
)
|
||||
|
||||
def _validate_mm_updates(
|
||||
self,
|
||||
@@ -1962,7 +1977,8 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
"This is likely because you forgot to include input "
|
||||
"placeholder tokens (e.g., `<image>`, `<|image_pad|>`) "
|
||||
"in the prompt. If the model has a chat template, make "
|
||||
"sure you have applied it before calling `LLM.generate`.")
|
||||
"sure you have applied it before calling `LLM.generate`."
|
||||
)
|
||||
|
||||
def _validate_mm_placeholders(
|
||||
self,
|
||||
@@ -1978,7 +1994,8 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
f"corresponding to {item_count} {modality} items, but "
|
||||
f"instead found {len(placeholders)} prompt placeholders! "
|
||||
"Make sure the implementation of `_call_hf_processor` and "
|
||||
"`_get_mm_fields_config` are consistent with each other.")
|
||||
"`_get_mm_fields_config` are consistent with each other."
|
||||
)
|
||||
|
||||
def _maybe_apply_prompt_updates(
|
||||
self,
|
||||
@@ -2070,7 +2087,6 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
|
||||
|
||||
class EncDecMultiModalProcessor(BaseMultiModalProcessor[_I]):
|
||||
|
||||
@abstractmethod
|
||||
def create_encoder_prompt(
|
||||
self,
|
||||
@@ -2104,15 +2120,16 @@ class EncDecMultiModalProcessor(BaseMultiModalProcessor[_I]):
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
decoder_prompt_raw = self.create_decoder_prompt(prompt, mm_data)
|
||||
if isinstance(decoder_prompt_raw, str):
|
||||
decoder_prompt_ids = encode_tokens(tokenizer,
|
||||
decoder_prompt_raw,
|
||||
add_special_tokens=False)
|
||||
decoder_prompt_ids = encode_tokens(
|
||||
tokenizer, decoder_prompt_raw, add_special_tokens=False
|
||||
)
|
||||
else:
|
||||
decoder_prompt_ids = decoder_prompt_raw
|
||||
|
||||
mm_inputs = MultiModalEncDecInputs(
|
||||
encoder_prompt_token_ids=encoder_inputs["prompt_token_ids"],
|
||||
**encoder_inputs)
|
||||
**encoder_inputs,
|
||||
)
|
||||
mm_inputs["prompt_token_ids"] = decoder_prompt_ids
|
||||
return mm_inputs
|
||||
|
||||
|
||||
@@ -10,15 +10,26 @@ import numpy.typing as npt
|
||||
from PIL import Image
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.config.multimodal import (AudioDummyOptions, BaseDummyOptions,
|
||||
ImageDummyOptions, VideoDummyOptions)
|
||||
from vllm.config.multimodal import (
|
||||
AudioDummyOptions,
|
||||
BaseDummyOptions,
|
||||
ImageDummyOptions,
|
||||
VideoDummyOptions,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
|
||||
from .inputs import (MultiModalDataDict, MultiModalEncDecInputs,
|
||||
MultiModalInputs, MultiModalKwargsItems,
|
||||
MultiModalPlaceholderDict)
|
||||
from .processing import (BaseMultiModalProcessor, BaseProcessingInfo,
|
||||
EncDecMultiModalProcessor)
|
||||
from .inputs import (
|
||||
MultiModalDataDict,
|
||||
MultiModalEncDecInputs,
|
||||
MultiModalInputs,
|
||||
MultiModalKwargsItems,
|
||||
MultiModalPlaceholderDict,
|
||||
)
|
||||
from .processing import (
|
||||
BaseMultiModalProcessor,
|
||||
BaseProcessingInfo,
|
||||
EncDecMultiModalProcessor,
|
||||
)
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -29,6 +40,7 @@ class ProcessorInputs:
|
||||
Represents the keyword arguments to
|
||||
[`vllm.multimodal.processing.BaseMultiModalProcessor.apply`][].
|
||||
"""
|
||||
|
||||
prompt: Union[str, list[int]]
|
||||
mm_data: MultiModalDataDict
|
||||
hf_processor_mm_kwargs: Mapping[str, object] = field(default_factory=dict)
|
||||
@@ -86,7 +98,7 @@ class BaseDummyInputsBuilder(ABC, Generic[_I]):
|
||||
mm_counts: Count of items per modality
|
||||
mm_options: Configurable options per modality (optional).
|
||||
If None, use model defaults for backward compatibility.
|
||||
If provided, models can use these to customize dummy
|
||||
If provided, models can use these to customize dummy
|
||||
data generation.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
@@ -113,9 +125,11 @@ class BaseDummyInputsBuilder(ABC, Generic[_I]):
|
||||
|
||||
tokenization_kwargs = {"truncation": False}
|
||||
|
||||
return ProcessorInputs(prompt=dummy_text,
|
||||
mm_data=dummy_mm_data,
|
||||
tokenization_kwargs=tokenization_kwargs)
|
||||
return ProcessorInputs(
|
||||
prompt=dummy_text,
|
||||
mm_data=dummy_mm_data,
|
||||
tokenization_kwargs=tokenization_kwargs,
|
||||
)
|
||||
|
||||
def _get_dummy_audios(
|
||||
self,
|
||||
@@ -130,10 +144,12 @@ class BaseDummyInputsBuilder(ABC, Generic[_I]):
|
||||
if overrides.length > length:
|
||||
logger.warning(
|
||||
"audio.length override (%d) exceeds model's "
|
||||
"maximum length (%d), will be ignored", overrides.length,
|
||||
length)
|
||||
"maximum length (%d), will be ignored",
|
||||
overrides.length,
|
||||
length,
|
||||
)
|
||||
length = min(length, overrides.length)
|
||||
audio = np.zeros((length, ))
|
||||
audio = np.zeros((length,))
|
||||
return [audio] * num_audios
|
||||
|
||||
def _get_dummy_images(
|
||||
@@ -151,15 +167,19 @@ class BaseDummyInputsBuilder(ABC, Generic[_I]):
|
||||
if overrides.width > width:
|
||||
logger.warning(
|
||||
"image.width override (%d) exceeds model's "
|
||||
"maximum width (%d), will be ignored", overrides.width,
|
||||
width)
|
||||
"maximum width (%d), will be ignored",
|
||||
overrides.width,
|
||||
width,
|
||||
)
|
||||
width = min(width, overrides.width)
|
||||
if overrides.height:
|
||||
if overrides.height > height:
|
||||
logger.warning(
|
||||
"image.height override (%d) exceeds model's "
|
||||
"maximum height (%d), will be ignored",
|
||||
overrides.height, height)
|
||||
overrides.height,
|
||||
height,
|
||||
)
|
||||
height = min(height, overrides.height)
|
||||
image = Image.new("RGB", (width, height), color=255)
|
||||
return [image] * num_images
|
||||
@@ -181,21 +201,27 @@ class BaseDummyInputsBuilder(ABC, Generic[_I]):
|
||||
logger.warning(
|
||||
"video.num_frames override (%d) exceeds model's "
|
||||
"maximum number of frames (%d), will be ignored",
|
||||
overrides.num_frames, num_frames)
|
||||
overrides.num_frames,
|
||||
num_frames,
|
||||
)
|
||||
num_frames = min(num_frames, overrides.num_frames)
|
||||
if overrides.width:
|
||||
if overrides.width > width:
|
||||
logger.warning(
|
||||
"video.width override (%d) exceeds model's "
|
||||
"maximum width (%d), will be ignored", overrides.width,
|
||||
width)
|
||||
"maximum width (%d), will be ignored",
|
||||
overrides.width,
|
||||
width,
|
||||
)
|
||||
width = min(width, overrides.width)
|
||||
if overrides.height:
|
||||
if overrides.height > height:
|
||||
logger.warning(
|
||||
"video.height override (%d) exceeds model's "
|
||||
"maximum height (%d), will be ignored",
|
||||
overrides.height, height)
|
||||
overrides.height,
|
||||
height,
|
||||
)
|
||||
height = min(height, overrides.height)
|
||||
video = np.full((num_frames, width, height, 3), 255)
|
||||
return [video] * num_videos
|
||||
@@ -236,7 +262,8 @@ class MultiModalProfiler(Generic[_I]):
|
||||
|
||||
factory = self.dummy_inputs
|
||||
processor_inputs = factory.get_dummy_processor_inputs(
|
||||
seq_len, mm_counts, mm_options)
|
||||
seq_len, mm_counts, mm_options
|
||||
)
|
||||
|
||||
return self.processor.apply(
|
||||
prompt=processor_inputs.prompt,
|
||||
@@ -253,9 +280,10 @@ class MultiModalProfiler(Generic[_I]):
|
||||
placeholders_by_modality = mm_inputs["mm_placeholders"]
|
||||
|
||||
return {
|
||||
modality:
|
||||
sum(item.get_num_embeds() if mm_embeddings_only else item.length
|
||||
for item in placeholders)
|
||||
modality: sum(
|
||||
item.get_num_embeds() if mm_embeddings_only else item.length
|
||||
for item in placeholders
|
||||
)
|
||||
for modality, placeholders in placeholders_by_modality.items()
|
||||
}
|
||||
|
||||
@@ -330,8 +358,7 @@ class MultiModalProfiler(Generic[_I]):
|
||||
return max_tokens_per_item
|
||||
|
||||
mm_inputs = self._get_dummy_mm_inputs(seq_len, mm_counts)
|
||||
return self._get_mm_num_tokens(mm_inputs,
|
||||
mm_embeddings_only=mm_embeddings_only)
|
||||
return self._get_mm_num_tokens(mm_inputs, mm_embeddings_only=mm_embeddings_only)
|
||||
|
||||
def get_mm_max_contiguous_tokens(
|
||||
self,
|
||||
@@ -349,6 +376,4 @@ class MultiModalProfiler(Generic[_I]):
|
||||
initializing the encoder cache size.
|
||||
"""
|
||||
|
||||
return self._get_mm_max_tokens(seq_len,
|
||||
mm_counts,
|
||||
mm_embeddings_only=False)
|
||||
return self._get_mm_max_tokens(seq_len, mm_counts, mm_embeddings_only=False)
|
||||
|
||||
+36
-34
@@ -8,15 +8,21 @@ import torch.nn as nn
|
||||
|
||||
from vllm.config.multimodal import BaseDummyOptions
|
||||
from vllm.logger import init_logger
|
||||
from vllm.transformers_utils.tokenizer import (AnyTokenizer,
|
||||
cached_tokenizer_from_config)
|
||||
from vllm.transformers_utils.tokenizer import AnyTokenizer, cached_tokenizer_from_config
|
||||
from vllm.utils import ClassRegistry
|
||||
|
||||
from .cache import BaseMultiModalProcessorCache
|
||||
from .processing import (BaseMultiModalProcessor, BaseProcessingInfo,
|
||||
InputProcessingContext)
|
||||
from .profiling import (BaseDummyInputsBuilder, DummyDecoderData,
|
||||
DummyEncoderData, MultiModalProfiler)
|
||||
from .processing import (
|
||||
BaseMultiModalProcessor,
|
||||
BaseProcessingInfo,
|
||||
InputProcessingContext,
|
||||
)
|
||||
from .profiling import (
|
||||
BaseDummyInputsBuilder,
|
||||
DummyDecoderData,
|
||||
DummyEncoderData,
|
||||
MultiModalProfiler,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config import ModelConfig
|
||||
@@ -38,8 +44,7 @@ class ProcessingInfoFactory(Protocol[_I_co]):
|
||||
def __call__(
|
||||
self,
|
||||
ctx: InputProcessingContext,
|
||||
) -> _I_co:
|
||||
...
|
||||
) -> _I_co: ...
|
||||
|
||||
|
||||
class DummyInputsBuilderFactory(Protocol[_I]): # type: ignore[misc]
|
||||
@@ -49,8 +54,7 @@ class DummyInputsBuilderFactory(Protocol[_I]): # type: ignore[misc]
|
||||
instance from the context.
|
||||
"""
|
||||
|
||||
def __call__(self, info: _I) -> BaseDummyInputsBuilder[_I]:
|
||||
...
|
||||
def __call__(self, info: _I) -> BaseDummyInputsBuilder[_I]: ...
|
||||
|
||||
|
||||
class MultiModalProcessorFactory(Protocol[_I]): # type: ignore[misc]
|
||||
@@ -66,8 +70,7 @@ class MultiModalProcessorFactory(Protocol[_I]): # type: ignore[misc]
|
||||
dummy_inputs: BaseDummyInputsBuilder[_I],
|
||||
*,
|
||||
cache: Optional[BaseMultiModalProcessorCache] = None,
|
||||
) -> BaseMultiModalProcessor[_I]:
|
||||
...
|
||||
) -> BaseMultiModalProcessor[_I]: ...
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
@@ -93,8 +96,7 @@ class MultiModalRegistry:
|
||||
"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._processor_factories = ClassRegistry[nn.Module,
|
||||
_ProcessorFactories]()
|
||||
self._processor_factories = ClassRegistry[nn.Module, _ProcessorFactories]()
|
||||
|
||||
def _extract_mm_options(
|
||||
self,
|
||||
@@ -112,8 +114,7 @@ class MultiModalRegistry:
|
||||
mm_options = {
|
||||
m: opt
|
||||
for m in model_config.multimodal_config.limit_per_prompt
|
||||
if (opt := model_config.multimodal_config.get_dummy_options(m)
|
||||
) is not None
|
||||
if (opt := model_config.multimodal_config.get_dummy_options(m)) is not None
|
||||
}
|
||||
|
||||
return mm_options if len(mm_options) > 0 else None
|
||||
@@ -121,8 +122,8 @@ class MultiModalRegistry:
|
||||
def supports_multimodal_inputs(self, model_config: "ModelConfig") -> bool:
|
||||
"""
|
||||
Checks if the model supports multimodal inputs.
|
||||
Returns True if the model is multimodal with any non-zero supported
|
||||
modalities, otherwise returns False, effectively running in
|
||||
Returns True if the model is multimodal with any non-zero supported
|
||||
modalities, otherwise returns False, effectively running in
|
||||
text-only mode.
|
||||
"""
|
||||
if not model_config.is_multimodal_model:
|
||||
@@ -135,11 +136,13 @@ class MultiModalRegistry:
|
||||
|
||||
# Check if all supported modalities have limit == 0
|
||||
if all(
|
||||
mm_config.get_limit_per_prompt(modality) == 0
|
||||
for modality in supported_modalities):
|
||||
mm_config.get_limit_per_prompt(modality) == 0
|
||||
for modality in supported_modalities
|
||||
):
|
||||
logger.info_once(
|
||||
"All limits of multimodal modalities supported by the model "
|
||||
"are set to 0, running in text-only mode.")
|
||||
"are set to 0, running in text-only mode."
|
||||
)
|
||||
return False
|
||||
|
||||
return True
|
||||
@@ -165,10 +168,7 @@ class MultiModalRegistry:
|
||||
|
||||
return profiler.get_mm_max_contiguous_tokens(
|
||||
seq_len,
|
||||
{
|
||||
modality: 1
|
||||
for modality, limit in mm_limits.items() if limit > 0
|
||||
},
|
||||
{modality: 1 for modality, limit in mm_limits.items() if limit > 0},
|
||||
)
|
||||
|
||||
def get_max_tokens_per_item_by_nonzero_modality(
|
||||
@@ -235,7 +235,9 @@ class MultiModalRegistry:
|
||||
logger.warning(
|
||||
"Model class %s already has a multi-modal processor "
|
||||
"registered to %s. It is overwritten by the new one.",
|
||||
model_cls, self)
|
||||
model_cls,
|
||||
self,
|
||||
)
|
||||
|
||||
self._processor_factories[model_cls] = _ProcessorFactories(
|
||||
info=info,
|
||||
@@ -315,15 +317,15 @@ class MultiModalRegistry:
|
||||
# count-only behavior remains unchanged.
|
||||
mm_options = self._extract_mm_options(model_config)
|
||||
|
||||
dummy_data = profiler.get_decoder_dummy_data(seq_len, mm_counts,
|
||||
mm_options)
|
||||
dummy_data = profiler.get_decoder_dummy_data(seq_len, mm_counts, mm_options)
|
||||
|
||||
# Having more tokens is over-conservative but otherwise fine
|
||||
token_ids = dummy_data.prompt_token_ids
|
||||
if len(token_ids) < seq_len:
|
||||
raise AssertionError(
|
||||
f"Expected at least {seq_len} dummy tokens for profiling, "
|
||||
f"but found {len(token_ids)} tokens instead.")
|
||||
f"but found {len(token_ids)} tokens instead."
|
||||
)
|
||||
|
||||
return dummy_data
|
||||
|
||||
@@ -348,8 +350,7 @@ class MultiModalRegistry:
|
||||
# count-only behavior remains unchanged.
|
||||
mm_options = self._extract_mm_options(model_config)
|
||||
|
||||
dummy_data = profiler.get_encoder_dummy_data(seq_len, mm_counts,
|
||||
mm_options)
|
||||
dummy_data = profiler.get_encoder_dummy_data(seq_len, mm_counts, mm_options)
|
||||
|
||||
# Having more tokens is over-conservative but otherwise fine
|
||||
token_ids = dummy_data.prompt_token_ids
|
||||
@@ -368,15 +369,16 @@ class MultiModalRegistry:
|
||||
"""
|
||||
if not model_config.is_encoder_decoder:
|
||||
return 0
|
||||
max_tokens = self.\
|
||||
get_max_tokens_per_item_by_nonzero_modality(model_config)
|
||||
max_tokens = self.get_max_tokens_per_item_by_nonzero_modality(model_config)
|
||||
if not max_tokens:
|
||||
# TODO - this function assumes encoder-decoder models are
|
||||
# multimodal. This will need to change when adding support for more
|
||||
# than whisper.
|
||||
return 0
|
||||
assert len(max_tokens) == 1, "Encoder-decoder models are expected \
|
||||
assert len(max_tokens) == 1, (
|
||||
"Encoder-decoder models are expected \
|
||||
to implement the multimodal interface with at most one modality."
|
||||
)
|
||||
|
||||
first_modality = next(iter(max_tokens))
|
||||
return max_tokens[first_modality]
|
||||
|
||||
+65
-52
@@ -28,8 +28,12 @@ from .video import VideoMediaIO
|
||||
_M = TypeVar("_M")
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from .inputs import (BatchedTensorInputs, MultiModalKwargsItem,
|
||||
MultiModalKwargsItems, MultiModalPlaceholderDict)
|
||||
from .inputs import (
|
||||
BatchedTensorInputs,
|
||||
MultiModalKwargsItem,
|
||||
MultiModalKwargsItems,
|
||||
MultiModalPlaceholderDict,
|
||||
)
|
||||
else:
|
||||
BatchedTensorInputs = Any
|
||||
MultiModalKwargsItem = Any
|
||||
@@ -37,12 +41,12 @@ else:
|
||||
MultiModalPlaceholderDict = Any
|
||||
|
||||
global_thread_pool = ThreadPoolExecutor(
|
||||
max_workers=envs.VLLM_MEDIA_LOADING_THREAD_COUNT)
|
||||
max_workers=envs.VLLM_MEDIA_LOADING_THREAD_COUNT
|
||||
)
|
||||
atexit.register(global_thread_pool.shutdown)
|
||||
|
||||
|
||||
class MediaConnector:
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
media_io_kwargs: Optional[dict[str, dict[str, Any]]] = None,
|
||||
@@ -53,9 +57,9 @@ class MediaConnector:
|
||||
) -> None:
|
||||
"""
|
||||
Args:
|
||||
media_io_kwargs: Additional args passed to process media
|
||||
inputs, keyed by modalities. For example,
|
||||
to set num_frames for video, set
|
||||
media_io_kwargs: Additional args passed to process media
|
||||
inputs, keyed by modalities. For example,
|
||||
to set num_frames for video, set
|
||||
`--media-io-kwargs '{"video":{"num_frames":40}}'`
|
||||
connection: HTTP connection client to download media contents.
|
||||
allowed_local_media_path: A local directory to load media files
|
||||
@@ -63,8 +67,9 @@ class MediaConnector:
|
||||
"""
|
||||
super().__init__()
|
||||
|
||||
self.media_io_kwargs: dict[str, dict[
|
||||
str, Any]] = media_io_kwargs if media_io_kwargs else {}
|
||||
self.media_io_kwargs: dict[str, dict[str, Any]] = (
|
||||
media_io_kwargs if media_io_kwargs else {}
|
||||
)
|
||||
self.connection = connection
|
||||
|
||||
if allowed_local_media_path:
|
||||
@@ -73,11 +78,13 @@ class MediaConnector:
|
||||
if not allowed_local_media_path_.exists():
|
||||
raise ValueError(
|
||||
"Invalid `--allowed-local-media-path`: The path "
|
||||
f"{allowed_local_media_path_} does not exist.")
|
||||
f"{allowed_local_media_path_} does not exist."
|
||||
)
|
||||
if not allowed_local_media_path_.is_dir():
|
||||
raise ValueError(
|
||||
"Invalid `--allowed-local-media-path`: The path "
|
||||
f"{allowed_local_media_path_} must be a directory.")
|
||||
f"{allowed_local_media_path_} must be a directory."
|
||||
)
|
||||
else:
|
||||
allowed_local_media_path_ = None
|
||||
|
||||
@@ -107,24 +114,29 @@ class MediaConnector:
|
||||
) -> _M: # type: ignore[type-var]
|
||||
allowed_local_media_path = self.allowed_local_media_path
|
||||
if allowed_local_media_path is None:
|
||||
raise RuntimeError("Cannot load local files without "
|
||||
"`--allowed-local-media-path`.")
|
||||
raise RuntimeError(
|
||||
"Cannot load local files without `--allowed-local-media-path`."
|
||||
)
|
||||
|
||||
filepath = Path(url2pathname(url_spec.path))
|
||||
if allowed_local_media_path not in filepath.resolve().parents:
|
||||
raise ValueError(
|
||||
f"The file path {filepath} must be a subpath "
|
||||
f"of `--allowed-local-media-path` {allowed_local_media_path}.")
|
||||
f"of `--allowed-local-media-path` {allowed_local_media_path}."
|
||||
)
|
||||
|
||||
return media_io.load_file(filepath)
|
||||
|
||||
def _assert_url_in_allowed_media_domains(self, url_spec) -> None:
|
||||
if self.allowed_media_domains and url_spec.hostname not in \
|
||||
self.allowed_media_domains:
|
||||
if (
|
||||
self.allowed_media_domains
|
||||
and url_spec.hostname not in self.allowed_media_domains
|
||||
):
|
||||
raise ValueError(
|
||||
f"The URL must be from one of the allowed domains: "
|
||||
f"{self.allowed_media_domains}. Input URL domain: "
|
||||
f"{url_spec.hostname}")
|
||||
f"{url_spec.hostname}"
|
||||
)
|
||||
|
||||
def load_from_url(
|
||||
self,
|
||||
@@ -175,20 +187,19 @@ class MediaConnector:
|
||||
timeout=fetch_timeout,
|
||||
allow_redirects=envs.VLLM_MEDIA_URL_ALLOW_REDIRECTS,
|
||||
)
|
||||
future = loop.run_in_executor(global_thread_pool,
|
||||
media_io.load_bytes, data)
|
||||
future = loop.run_in_executor(global_thread_pool, media_io.load_bytes, data)
|
||||
return await future
|
||||
|
||||
if url_spec.scheme == "data":
|
||||
future = loop.run_in_executor(global_thread_pool,
|
||||
self._load_data_url, url_spec,
|
||||
media_io)
|
||||
future = loop.run_in_executor(
|
||||
global_thread_pool, self._load_data_url, url_spec, media_io
|
||||
)
|
||||
return await future
|
||||
|
||||
if url_spec.scheme == "file":
|
||||
future = loop.run_in_executor(global_thread_pool,
|
||||
self._load_file_url, url_spec,
|
||||
media_io)
|
||||
future = loop.run_in_executor(
|
||||
global_thread_pool, self._load_file_url, url_spec, media_io
|
||||
)
|
||||
return await future
|
||||
msg = "The URL must be either a HTTP, data or file URL."
|
||||
raise ValueError(msg)
|
||||
@@ -234,8 +245,9 @@ class MediaConnector:
|
||||
|
||||
By default, the image is converted into RGB format.
|
||||
"""
|
||||
image_io = ImageMediaIO(image_mode=image_mode,
|
||||
**self.media_io_kwargs.get("image", {}))
|
||||
image_io = ImageMediaIO(
|
||||
image_mode=image_mode, **self.media_io_kwargs.get("image", {})
|
||||
)
|
||||
|
||||
try:
|
||||
return self.load_from_url(
|
||||
@@ -258,8 +270,9 @@ class MediaConnector:
|
||||
|
||||
By default, the image is converted into RGB format.
|
||||
"""
|
||||
image_io = ImageMediaIO(image_mode=image_mode,
|
||||
**self.media_io_kwargs.get("image", {}))
|
||||
image_io = ImageMediaIO(
|
||||
image_mode=image_mode, **self.media_io_kwargs.get("image", {})
|
||||
)
|
||||
|
||||
try:
|
||||
return await self.load_from_url_async(
|
||||
@@ -280,10 +293,10 @@ class MediaConnector:
|
||||
"""
|
||||
Load video from an HTTP or base64 data URL.
|
||||
"""
|
||||
image_io = ImageMediaIO(image_mode=image_mode,
|
||||
**self.media_io_kwargs.get("image", {}))
|
||||
video_io = VideoMediaIO(image_io,
|
||||
**self.media_io_kwargs.get("video", {}))
|
||||
image_io = ImageMediaIO(
|
||||
image_mode=image_mode, **self.media_io_kwargs.get("image", {})
|
||||
)
|
||||
video_io = VideoMediaIO(image_io, **self.media_io_kwargs.get("video", {}))
|
||||
|
||||
return self.load_from_url(
|
||||
video_url,
|
||||
@@ -302,10 +315,10 @@ class MediaConnector:
|
||||
|
||||
By default, the image is converted into RGB format.
|
||||
"""
|
||||
image_io = ImageMediaIO(image_mode=image_mode,
|
||||
**self.media_io_kwargs.get("image", {}))
|
||||
video_io = VideoMediaIO(image_io,
|
||||
**self.media_io_kwargs.get("video", {}))
|
||||
image_io = ImageMediaIO(
|
||||
image_mode=image_mode, **self.media_io_kwargs.get("image", {})
|
||||
)
|
||||
video_io = VideoMediaIO(image_io, **self.media_io_kwargs.get("video", {}))
|
||||
|
||||
return await self.load_from_url_async(
|
||||
video_url,
|
||||
@@ -356,7 +369,8 @@ def encode_video_base64(frames: npt.NDArray) -> str:
|
||||
|
||||
|
||||
def argsort_mm_positions(
|
||||
mm_positions: MultiModalPlaceholderDict) -> list[tuple[str, int]]:
|
||||
mm_positions: MultiModalPlaceholderDict,
|
||||
) -> list[tuple[str, int]]:
|
||||
"""
|
||||
Given a `MultiModalPlaceholderDict`, output a sequence of keys to
|
||||
sort the dictionary by `offset` (starting index in the input sequence)
|
||||
@@ -366,9 +380,11 @@ def argsort_mm_positions(
|
||||
A list of `(modality, idx)`, which can be used to access an item
|
||||
by `mm_positions[modality][idx]`.
|
||||
"""
|
||||
flat_items = ((modality, idx, item)
|
||||
for modality, items in mm_positions.items()
|
||||
for idx, item in enumerate(items))
|
||||
flat_items = (
|
||||
(modality, idx, item)
|
||||
for modality, items in mm_positions.items()
|
||||
for idx, item in enumerate(items)
|
||||
)
|
||||
|
||||
sorted_flat_items = sorted(flat_items, key=lambda x: x[2].offset)
|
||||
|
||||
@@ -397,7 +413,8 @@ def group_mm_kwargs_by_modality(
|
||||
raise RuntimeError(
|
||||
"`group_mm_kwargs_by_modality` now requires "
|
||||
"`merge_by_field_config` arg, please update your model runner "
|
||||
"according to https://github.com/vllm-project/vllm/pull/25676.")
|
||||
"according to https://github.com/vllm-project/vllm/pull/25676."
|
||||
)
|
||||
|
||||
from vllm.multimodal.inputs import MultiModalKwargs, MultiModalKwargsItems
|
||||
|
||||
@@ -409,7 +426,9 @@ def group_mm_kwargs_by_modality(
|
||||
if merge_by_field_config:
|
||||
mm_kwargs_group: BatchedTensorInputs = dict(
|
||||
MultiModalKwargsItems.from_seq(items_lst).get_data(
|
||||
pin_memory=pin_memory))
|
||||
pin_memory=pin_memory
|
||||
)
|
||||
)
|
||||
|
||||
if device is not None:
|
||||
mm_kwargs_group = json_map_leaves(
|
||||
@@ -440,9 +459,7 @@ def fetch_audio(
|
||||
audio_url: URL of the audio file to fetch.
|
||||
audio_io_kwargs: Additional kwargs passed to handle audio IO.
|
||||
"""
|
||||
media_io_kwargs = None if not audio_io_kwargs else {
|
||||
"audio": audio_io_kwargs
|
||||
}
|
||||
media_io_kwargs = None if not audio_io_kwargs else {"audio": audio_io_kwargs}
|
||||
media_connector = MediaConnector(media_io_kwargs=media_io_kwargs)
|
||||
return media_connector.fetch_audio(audio_url)
|
||||
|
||||
@@ -456,9 +473,7 @@ def fetch_image(
|
||||
image_url: URL of the image file to fetch.
|
||||
image_io_kwargs: Additional kwargs passed to handle image IO.
|
||||
"""
|
||||
media_io_kwargs = None if not image_io_kwargs else {
|
||||
"image": image_io_kwargs
|
||||
}
|
||||
media_io_kwargs = None if not image_io_kwargs else {"image": image_io_kwargs}
|
||||
media_connector = MediaConnector(media_io_kwargs=media_io_kwargs)
|
||||
return media_connector.fetch_image(image_url)
|
||||
|
||||
@@ -472,8 +487,6 @@ def fetch_video(
|
||||
video_url: URL of the video file to fetch.
|
||||
video_io_kwargs: Additional kwargs passed to handle video IO.
|
||||
"""
|
||||
media_io_kwargs = None if not video_io_kwargs else {
|
||||
"video": video_io_kwargs
|
||||
}
|
||||
media_io_kwargs = None if not video_io_kwargs else {"video": video_io_kwargs}
|
||||
media_connector = MediaConnector(media_io_kwargs=media_io_kwargs)
|
||||
return media_connector.fetch_video(video_url)
|
||||
|
||||
+40
-46
@@ -21,8 +21,9 @@ from .image import ImageMediaIO
|
||||
def resize_video(frames: npt.NDArray, size: tuple[int, int]) -> npt.NDArray:
|
||||
num_frames, _, _, channels = frames.shape
|
||||
new_height, new_width = size
|
||||
resized_frames = np.empty((num_frames, new_height, new_width, channels),
|
||||
dtype=frames.dtype)
|
||||
resized_frames = np.empty(
|
||||
(num_frames, new_height, new_width, channels), dtype=frames.dtype
|
||||
)
|
||||
# lazy import cv2 to avoid bothering users who only use text models
|
||||
import cv2
|
||||
|
||||
@@ -40,8 +41,7 @@ def rescale_video_size(frames: npt.NDArray, size_factor: float) -> npt.NDArray:
|
||||
return resize_video(frames, (new_height, new_width))
|
||||
|
||||
|
||||
def sample_frames_from_video(frames: npt.NDArray,
|
||||
num_frames: int) -> npt.NDArray:
|
||||
def sample_frames_from_video(frames: npt.NDArray, num_frames: int) -> npt.NDArray:
|
||||
total_frames = frames.shape[0]
|
||||
if num_frames == -1:
|
||||
return frames
|
||||
@@ -52,23 +52,19 @@ def sample_frames_from_video(frames: npt.NDArray,
|
||||
|
||||
|
||||
class VideoLoader:
|
||||
|
||||
@classmethod
|
||||
@abstractmethod
|
||||
def load_bytes(cls,
|
||||
data: bytes,
|
||||
num_frames: int = -1,
|
||||
**kwargs) -> tuple[npt.NDArray, dict[str, Any]]:
|
||||
def load_bytes(
|
||||
cls, data: bytes, num_frames: int = -1, **kwargs
|
||||
) -> tuple[npt.NDArray, dict[str, Any]]:
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
class VideoLoaderRegistry:
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.name2class: dict[str, type] = {}
|
||||
|
||||
def register(self, name: str):
|
||||
|
||||
def wrap(cls_to_register):
|
||||
self.name2class[name] = cls_to_register
|
||||
return cls_to_register
|
||||
@@ -87,7 +83,6 @@ VIDEO_LOADER_REGISTRY = VideoLoaderRegistry()
|
||||
|
||||
@VIDEO_LOADER_REGISTRY.register("opencv")
|
||||
class OpenCVVideoBackend(VideoLoader):
|
||||
|
||||
def get_cv2_video_api(self):
|
||||
import cv2.videoio_registry as vr
|
||||
|
||||
@@ -127,10 +122,9 @@ class OpenCVVideoBackend(VideoLoader):
|
||||
num_frames = total_frames_num
|
||||
frame_idx = list(range(0, num_frames))
|
||||
else:
|
||||
uniform_sampled_frames = np.linspace(0,
|
||||
total_frames_num - 1,
|
||||
num_frames,
|
||||
dtype=int)
|
||||
uniform_sampled_frames = np.linspace(
|
||||
0, total_frames_num - 1, num_frames, dtype=int
|
||||
)
|
||||
frame_idx = uniform_sampled_frames.tolist()
|
||||
|
||||
width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))
|
||||
@@ -148,8 +142,10 @@ class OpenCVVideoBackend(VideoLoader):
|
||||
frames[i] = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
|
||||
i += 1
|
||||
|
||||
assert i == num_frames, (f"Expected reading {num_frames} frames, "
|
||||
f"but only loaded {i} frames from video.")
|
||||
assert i == num_frames, (
|
||||
f"Expected reading {num_frames} frames, "
|
||||
f"but only loaded {i} frames from video."
|
||||
)
|
||||
|
||||
# Use transformers transformers.video_utils.VideoMetadata format
|
||||
# NOTE(Isotr0py): For models like Qwen3-VL/GLM4.5V, this metadata
|
||||
@@ -170,7 +166,6 @@ class OpenCVVideoBackend(VideoLoader):
|
||||
|
||||
@VIDEO_LOADER_REGISTRY.register("opencv_dynamic")
|
||||
class OpenCVDynamicVideoBackend(OpenCVVideoBackend):
|
||||
|
||||
@classmethod
|
||||
def load_bytes(
|
||||
cls,
|
||||
@@ -200,28 +195,28 @@ class OpenCVDynamicVideoBackend(OpenCVVideoBackend):
|
||||
frame_indices: Union[range, list[int]]
|
||||
if duration <= max_duration:
|
||||
n = int(math.floor(duration * fps))
|
||||
frame_indices = sorted({
|
||||
min(max_frame_idx, int(math.ceil(i * original_fps / fps)))
|
||||
for i in range(n)
|
||||
})
|
||||
frame_indices = sorted(
|
||||
{
|
||||
min(max_frame_idx, int(math.ceil(i * original_fps / fps)))
|
||||
for i in range(n)
|
||||
}
|
||||
)
|
||||
else:
|
||||
num_samples = int(max_duration * fps)
|
||||
if num_samples >= total_frames_num:
|
||||
frame_indices = range(total_frames_num)
|
||||
else:
|
||||
target_seconds = np.linspace(0,
|
||||
duration,
|
||||
num_samples,
|
||||
endpoint=True)
|
||||
frame_indices = sorted({
|
||||
min(max_frame_idx, int(math.ceil(t * original_fps)))
|
||||
for t in target_seconds
|
||||
})
|
||||
target_seconds = np.linspace(0, duration, num_samples, endpoint=True)
|
||||
frame_indices = sorted(
|
||||
{
|
||||
min(max_frame_idx, int(math.ceil(t * original_fps)))
|
||||
for t in target_seconds
|
||||
}
|
||||
)
|
||||
|
||||
width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))
|
||||
height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))
|
||||
frames = np.empty((len(frame_indices), height, width, 3),
|
||||
dtype=np.uint8)
|
||||
frames = np.empty((len(frame_indices), height, width, 3), dtype=np.uint8)
|
||||
|
||||
i = 0
|
||||
for idx in range(total_frames_num):
|
||||
@@ -236,7 +231,8 @@ class OpenCVDynamicVideoBackend(OpenCVVideoBackend):
|
||||
|
||||
assert i == len(frame_indices), (
|
||||
f"Expected reading {len(frame_indices)} frames, "
|
||||
f"but only loaded {i} frames from video.")
|
||||
f"but only loaded {i} frames from video."
|
||||
)
|
||||
|
||||
# Use transformers transformers.video_utils.VideoMetadata format
|
||||
metadata = {
|
||||
@@ -252,7 +248,6 @@ class OpenCVDynamicVideoBackend(OpenCVVideoBackend):
|
||||
|
||||
|
||||
class VideoMediaIO(MediaIO[npt.NDArray]):
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
image_io: ImageMediaIO,
|
||||
@@ -273,22 +268,22 @@ class VideoMediaIO(MediaIO[npt.NDArray]):
|
||||
self.video_loader = VIDEO_LOADER_REGISTRY.load(video_loader_backend)
|
||||
|
||||
def load_bytes(self, data: bytes) -> tuple[npt.NDArray, dict[str, Any]]:
|
||||
return self.video_loader.load_bytes(data,
|
||||
num_frames=self.num_frames,
|
||||
**self.kwargs)
|
||||
return self.video_loader.load_bytes(
|
||||
data, num_frames=self.num_frames, **self.kwargs
|
||||
)
|
||||
|
||||
def load_base64(self, media_type: str,
|
||||
data: str) -> tuple[npt.NDArray, dict[str, Any]]:
|
||||
def load_base64(
|
||||
self, media_type: str, data: str
|
||||
) -> tuple[npt.NDArray, dict[str, Any]]:
|
||||
if media_type.lower() == "video/jpeg":
|
||||
load_frame = partial(
|
||||
self.image_io.load_base64,
|
||||
"image/jpeg",
|
||||
)
|
||||
|
||||
return np.stack([
|
||||
np.asarray(load_frame(frame_data))
|
||||
for frame_data in data.split(",")
|
||||
]), {}
|
||||
return np.stack(
|
||||
[np.asarray(load_frame(frame_data)) for frame_data in data.split(",")]
|
||||
), {}
|
||||
|
||||
return self.load_bytes(base64.b64decode(data))
|
||||
|
||||
@@ -312,8 +307,7 @@ class VideoMediaIO(MediaIO[npt.NDArray]):
|
||||
image_format=video_format,
|
||||
)
|
||||
|
||||
return ",".join(
|
||||
encode_frame(Image.fromarray(frame)) for frame in video)
|
||||
return ",".join(encode_frame(Image.fromarray(frame)) for frame in video)
|
||||
|
||||
msg = "Only JPEG format is supported for now."
|
||||
raise NotImplementedError(msg)
|
||||
|
||||
Reference in New Issue
Block a user