[Perf] Hash videos by source bytes (#49607)

Signed-off-by: Guan-Ming (Wesley) Chiu <105915352+guan404ming@users.noreply.github.com>
Signed-off-by: Guan-Ming Chiu <105915352+guan404ming@users.noreply.github.com>
Co-authored-by: Isotr0py <2037008807@qq.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Guan-Ming Chiu
2026-07-27 23:57:55 +08:00
committed by GitHub
co-authored by Isotr0py mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent d2ca3002d9
commit 04502deca2
8 changed files with 94 additions and 21 deletions
+25
View File
@@ -9,6 +9,8 @@ import torch
from PIL import Image, ImageDraw
from vllm.multimodal.hasher import MultiModalHasher
from vllm.multimodal.media.base import MediaWithBytes
from vllm.multimodal.parse import MultiModalDataParser
pytestmark = pytest.mark.cpu_test
@@ -82,6 +84,29 @@ def test_hash_collision_array_shape():
assert hasher.hash_kwargs(data=arr1) != hasher.hash_kwargs(data=arr2)
def test_hash_collision_video_num_frames():
source = b"x" * 100
def item_for_hash(num_frames: int):
frames: np.ndarray = np.zeros((num_frames, 8, 8, 3), dtype=np.uint8)
metadata = {
"total_num_frames": 16,
"fps": 2.0,
"duration": 8.0,
"video_backend": "opencv",
"frames_indices": list(range(num_frames)),
"do_sample_frames": False,
}
video = MediaWithBytes((frames, metadata), source)
items = MultiModalDataParser()._parse_video_data([video])
return items.get_all_items_for_hash()[0]
hasher = MultiModalHasher
assert hasher.hash_kwargs(video=item_for_hash(2)) != hasher.hash_kwargs(
video=item_for_hash(4)
)
def test_hash_non_contiguous_array():
arr = np.arange(24).reshape(4, 6).T
assert not arr.flags.c_contiguous
+6
View File
@@ -83,6 +83,12 @@ class MultiModalHasher:
return cls.iter_item_to_bytes("image", obj.original_bytes)
if isinstance(obj, MediaWithBytes) and isinstance(obj.media, np.ndarray):
frames = obj.media
if frames.nbytes < len(obj.original_bytes):
return cls.iter_item_to_bytes("video", frames)
return cls.iter_item_to_bytes("video", obj.original_bytes)
if isinstance(obj, torch.Tensor):
tensor_obj: torch.Tensor = obj.cpu()
tensor_dtype = tensor_obj.dtype
+4 -1
View File
@@ -66,7 +66,10 @@ these are directly passed to the model without HF processing.
"""
VideoItem: TypeAlias = Union[
HfVideoItem, "torch.Tensor", tuple[HfVideoItem, dict[str, Any]]
HfVideoItem,
"torch.Tensor",
tuple[HfVideoItem, dict[str, Any]],
MediaWithBytes[tuple[HfVideoItem, dict[str, Any]]],
]
"""
A `transformers.video_utils.VideoInput` representing a single video item.
+12 -2
View File
@@ -2,9 +2,10 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from abc import ABC, abstractmethod
from collections.abc import Iterable, Iterator
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any, Generic, TypeVar
from typing import Any, Generic, TypeVar, cast
import numpy as np
@@ -22,7 +23,8 @@ class MediaWithBytes(Generic[_T]):
The wrapper delegates attribute access to the underlying media object,
making it behave transparently like the wrapped type (e.g., PIL.Image).
NOTE: Currently, this wrapper is used only for the image modality.
NOTE: Currently, this wrapper is used only for the image and video
modalities.
"""
media: _T
@@ -32,6 +34,14 @@ class MediaWithBytes(Generic[_T]):
"""Allow np.array(obj) to return np.array(obj.media)."""
return np.array(self.media, *args, **kwargs)
def __iter__(self) -> Iterator[Any]:
"""Allow unpacking obj to unpack obj.media (e.g. video tuples)."""
return iter(cast(Iterable[Any], self.media))
def __getitem__(self, index: Any) -> Any:
"""Allow obj[i] to index obj.media (e.g. video tuples)."""
return cast(Any, self.media)[index]
def __getstate__(self):
return self.__dict__.copy()
+3 -3
View File
@@ -29,7 +29,7 @@ from vllm.multimodal.video import get_video_loader_backend_for_processor
from vllm.utils.registry import ExtensionManager
from .audio import AudioEmbeddingMediaIO, AudioMediaIO
from .base import MediaIO
from .base import MediaIO, MediaWithBytes
from .image import ImageEmbeddingMediaIO, ImageMediaIO
from .video import VideoMediaIO
@@ -536,7 +536,7 @@ class MediaConnector:
*,
image_mode: str | None = "RGB",
video_processor: str | None = None,
) -> tuple[npt.NDArray, dict[str, Any]]:
) -> MediaWithBytes[tuple[npt.NDArray, dict[str, Any]]]:
"""
Load video from an HTTP or base64 data URL.
"""
@@ -562,7 +562,7 @@ class MediaConnector:
*,
image_mode: str | None = "RGB",
video_processor: str | None = None,
) -> tuple[npt.NDArray, dict[str, Any]]:
) -> MediaWithBytes[tuple[npt.NDArray, dict[str, Any]]]:
"""
Asynchronously load video from an HTTP or base64 data URL.
+12 -7
View File
@@ -13,13 +13,13 @@ from vllm import envs
from vllm.logger import init_logger
from ..video import VIDEO_LOADER_REGISTRY
from .base import MediaIO
from .base import MediaIO, MediaWithBytes
from .image import ImageMediaIO
logger = init_logger(__name__)
class VideoMediaIO(MediaIO[tuple[npt.NDArray, dict[str, Any]]]):
class VideoMediaIO(MediaIO[MediaWithBytes[tuple[npt.NDArray, dict[str, Any]]]]):
"""Configuration values can be user-provided either by --media-io-kwargs or
by the runtime API field "media_io_kwargs". Ensure proper validation and
error handling.
@@ -91,14 +91,17 @@ class VideoMediaIO(MediaIO[tuple[npt.NDArray, dict[str, Any]]]):
self.kwargs = kwargs
self.video_loader = VIDEO_LOADER_REGISTRY.load(video_loader_backend)
def load_bytes(self, data: bytes) -> tuple[npt.NDArray, dict[str, Any]]:
return self.video_loader.load_bytes(
def load_bytes(
self, data: bytes
) -> MediaWithBytes[tuple[npt.NDArray, dict[str, Any]]]:
video = self.video_loader.load_bytes(
data, num_frames=self.num_frames, **self.kwargs
)
return MediaWithBytes(video, data)
def load_base64(
self, media_type: str, data: str
) -> tuple[npt.NDArray, dict[str, Any]]:
) -> MediaWithBytes[tuple[npt.NDArray, dict[str, Any]]]:
if media_type.lower() == "video/jpeg":
load_frame = partial(
self.image_io.load_base64,
@@ -160,11 +163,13 @@ class VideoMediaIO(MediaIO[tuple[npt.NDArray, dict[str, Any]]]):
"frames_indices": frames_indices,
"do_sample_frames": self.kwargs.get("do_sample_frames", False),
}
return frames, metadata
return MediaWithBytes((frames, metadata), data.encode())
return self.load_bytes(pybase64.b64decode(data))
def load_file(self, filepath: Path) -> tuple[npt.NDArray, dict[str, Any]]:
def load_file(
self, filepath: Path
) -> MediaWithBytes[tuple[npt.NDArray, dict[str, Any]]]:
with filepath.open("rb") as f:
data = f.read()
+24 -6
View File
@@ -366,6 +366,20 @@ class VideoProcessorItems(ProcessorBatchItems[HfVideoItem | None]):
self.metadata = metadata
def _unwrap(self, item: Any) -> Any:
if isinstance(item, tuple):
frames, metadata = item
return super()._unwrap(frames), metadata
return super()._unwrap(item)
def get_item_for_hash(self, index: int) -> Any:
item = self.data[index]
if isinstance(item, MediaWithBytes) and isinstance(self.metadata, list):
metadata = self.metadata[index]
if metadata is not None:
return item, metadata
return item
def get_num_frames(self, item_idx: int) -> int:
video = self.get(item_idx)
if video is None:
@@ -552,7 +566,10 @@ class MultiModalDataParser:
def _get_video_with_metadata(
self,
video: VideoItem,
) -> tuple[np.ndarray, dict[str, Any] | None]:
) -> tuple[np.ndarray | MediaWithBytes[np.ndarray], dict[str, Any] | None]:
if isinstance(video, MediaWithBytes):
new_video, metadata = self._get_video_with_metadata(video.media)
return MediaWithBytes(new_video, video.original_bytes), metadata
if isinstance(video, tuple):
return video
if isinstance(video, list):
@@ -653,7 +670,11 @@ class MultiModalDataParser:
else:
data_items = data # type: ignore[assignment]
new_videos = list[tuple[np.ndarray, dict[str, Any] | None]]()
new_videos = list[
np.ndarray
| MediaWithBytes[np.ndarray]
| tuple[np.ndarray | MediaWithBytes[np.ndarray], dict[str, Any]]
]()
metadata_lst: list[dict[str, Any] | None] = []
for data_item in data_items:
video, metadata = self._get_video_with_metadata(data_item)
@@ -664,12 +685,9 @@ class MultiModalDataParser:
"Please check your video input in `multi_modal_data`"
)
new_videos.append((video, metadata))
metadata_lst.append(metadata)
else:
new_videos.append(video)
if not self.video_needs_metadata:
metadata = None
metadata_lst.append(metadata)
return VideoProcessorItems(new_videos, metadata=metadata_lst)
+8 -2
View File
@@ -24,7 +24,13 @@ from .inputs import (
MultiModalKwargsItem,
MultiModalSharedField,
)
from .media import AudioMediaIO, ImageMediaIO, MediaConnector, VideoMediaIO
from .media import (
AudioMediaIO,
ImageMediaIO,
MediaConnector,
MediaWithBytes,
VideoMediaIO,
)
if TYPE_CHECKING:
import torch.types
@@ -332,7 +338,7 @@ def fetch_image(
def fetch_video(
video_url: str,
video_io_kwargs: dict[str, Any] | None = None,
) -> tuple[npt.NDArray, dict[str, Any]]:
) -> MediaWithBytes[tuple[npt.NDArray, dict[str, Any]]]:
"""
Args:
video_url: URL of the video file to fetch.