From 04502deca2f66b874d0fceae446c668c987413c0 Mon Sep 17 00:00:00 2001 From: Guan-Ming Chiu <105915352+guan404ming@users.noreply.github.com> Date: Mon, 27 Jul 2026 23:57:55 +0800 Subject: [PATCH] [Perf] Hash videos by source bytes (#49607) Signed-off-by: Guan-Ming (Wesley) Chiu <105915352+guan404ming@users.noreply.github.com> Signed-off-by: Guan-Ming Chiu <105915352+guan404ming@users.noreply.github.com> Co-authored-by: Isotr0py <2037008807@qq.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- tests/multimodal/test_hasher.py | 25 +++++++++++++++++++++++++ vllm/multimodal/hasher.py | 6 ++++++ vllm/multimodal/inputs.py | 5 ++++- vllm/multimodal/media/base.py | 14 ++++++++++++-- vllm/multimodal/media/connector.py | 6 +++--- vllm/multimodal/media/video.py | 19 ++++++++++++------- vllm/multimodal/parse.py | 30 ++++++++++++++++++++++++------ vllm/multimodal/utils.py | 10 ++++++++-- 8 files changed, 94 insertions(+), 21 deletions(-) diff --git a/tests/multimodal/test_hasher.py b/tests/multimodal/test_hasher.py index fdedcaea27..f1ddc37875 100644 --- a/tests/multimodal/test_hasher.py +++ b/tests/multimodal/test_hasher.py @@ -9,6 +9,8 @@ import torch from PIL import Image, ImageDraw from vllm.multimodal.hasher import MultiModalHasher +from vllm.multimodal.media.base import MediaWithBytes +from vllm.multimodal.parse import MultiModalDataParser pytestmark = pytest.mark.cpu_test @@ -82,6 +84,29 @@ def test_hash_collision_array_shape(): assert hasher.hash_kwargs(data=arr1) != hasher.hash_kwargs(data=arr2) +def test_hash_collision_video_num_frames(): + source = b"x" * 100 + + def item_for_hash(num_frames: int): + frames: np.ndarray = np.zeros((num_frames, 8, 8, 3), dtype=np.uint8) + metadata = { + "total_num_frames": 16, + "fps": 2.0, + "duration": 8.0, + "video_backend": "opencv", + "frames_indices": list(range(num_frames)), + "do_sample_frames": False, + } + video = MediaWithBytes((frames, metadata), source) + items = MultiModalDataParser()._parse_video_data([video]) + return items.get_all_items_for_hash()[0] + + hasher = MultiModalHasher + assert hasher.hash_kwargs(video=item_for_hash(2)) != hasher.hash_kwargs( + video=item_for_hash(4) + ) + + def test_hash_non_contiguous_array(): arr = np.arange(24).reshape(4, 6).T assert not arr.flags.c_contiguous diff --git a/vllm/multimodal/hasher.py b/vllm/multimodal/hasher.py index 6caf9c1142..ba9bcd0ea8 100644 --- a/vllm/multimodal/hasher.py +++ b/vllm/multimodal/hasher.py @@ -83,6 +83,12 @@ class MultiModalHasher: return cls.iter_item_to_bytes("image", obj.original_bytes) + if isinstance(obj, MediaWithBytes) and isinstance(obj.media, np.ndarray): + frames = obj.media + if frames.nbytes < len(obj.original_bytes): + return cls.iter_item_to_bytes("video", frames) + return cls.iter_item_to_bytes("video", obj.original_bytes) + if isinstance(obj, torch.Tensor): tensor_obj: torch.Tensor = obj.cpu() tensor_dtype = tensor_obj.dtype diff --git a/vllm/multimodal/inputs.py b/vllm/multimodal/inputs.py index c55bbe4623..71f17a8648 100644 --- a/vllm/multimodal/inputs.py +++ b/vllm/multimodal/inputs.py @@ -66,7 +66,10 @@ these are directly passed to the model without HF processing. """ VideoItem: TypeAlias = Union[ - HfVideoItem, "torch.Tensor", tuple[HfVideoItem, dict[str, Any]] + HfVideoItem, + "torch.Tensor", + tuple[HfVideoItem, dict[str, Any]], + MediaWithBytes[tuple[HfVideoItem, dict[str, Any]]], ] """ A `transformers.video_utils.VideoInput` representing a single video item. diff --git a/vllm/multimodal/media/base.py b/vllm/multimodal/media/base.py index 91e7a49471..3c072a09a5 100644 --- a/vllm/multimodal/media/base.py +++ b/vllm/multimodal/media/base.py @@ -2,9 +2,10 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project from abc import ABC, abstractmethod +from collections.abc import Iterable, Iterator from dataclasses import dataclass, field from pathlib import Path -from typing import Any, Generic, TypeVar +from typing import Any, Generic, TypeVar, cast import numpy as np @@ -22,7 +23,8 @@ class MediaWithBytes(Generic[_T]): The wrapper delegates attribute access to the underlying media object, making it behave transparently like the wrapped type (e.g., PIL.Image). - NOTE: Currently, this wrapper is used only for the image modality. + NOTE: Currently, this wrapper is used only for the image and video + modalities. """ media: _T @@ -32,6 +34,14 @@ class MediaWithBytes(Generic[_T]): """Allow np.array(obj) to return np.array(obj.media).""" return np.array(self.media, *args, **kwargs) + def __iter__(self) -> Iterator[Any]: + """Allow unpacking obj to unpack obj.media (e.g. video tuples).""" + return iter(cast(Iterable[Any], self.media)) + + def __getitem__(self, index: Any) -> Any: + """Allow obj[i] to index obj.media (e.g. video tuples).""" + return cast(Any, self.media)[index] + def __getstate__(self): return self.__dict__.copy() diff --git a/vllm/multimodal/media/connector.py b/vllm/multimodal/media/connector.py index 656f537c5b..fed41657e9 100644 --- a/vllm/multimodal/media/connector.py +++ b/vllm/multimodal/media/connector.py @@ -29,7 +29,7 @@ from vllm.multimodal.video import get_video_loader_backend_for_processor from vllm.utils.registry import ExtensionManager from .audio import AudioEmbeddingMediaIO, AudioMediaIO -from .base import MediaIO +from .base import MediaIO, MediaWithBytes from .image import ImageEmbeddingMediaIO, ImageMediaIO from .video import VideoMediaIO @@ -536,7 +536,7 @@ class MediaConnector: *, image_mode: str | None = "RGB", video_processor: str | None = None, - ) -> tuple[npt.NDArray, dict[str, Any]]: + ) -> MediaWithBytes[tuple[npt.NDArray, dict[str, Any]]]: """ Load video from an HTTP or base64 data URL. """ @@ -562,7 +562,7 @@ class MediaConnector: *, image_mode: str | None = "RGB", video_processor: str | None = None, - ) -> tuple[npt.NDArray, dict[str, Any]]: + ) -> MediaWithBytes[tuple[npt.NDArray, dict[str, Any]]]: """ Asynchronously load video from an HTTP or base64 data URL. diff --git a/vllm/multimodal/media/video.py b/vllm/multimodal/media/video.py index 978cf06c23..124dcd7f7e 100644 --- a/vllm/multimodal/media/video.py +++ b/vllm/multimodal/media/video.py @@ -13,13 +13,13 @@ from vllm import envs from vllm.logger import init_logger from ..video import VIDEO_LOADER_REGISTRY -from .base import MediaIO +from .base import MediaIO, MediaWithBytes from .image import ImageMediaIO logger = init_logger(__name__) -class VideoMediaIO(MediaIO[tuple[npt.NDArray, dict[str, Any]]]): +class VideoMediaIO(MediaIO[MediaWithBytes[tuple[npt.NDArray, dict[str, Any]]]]): """Configuration values can be user-provided either by --media-io-kwargs or by the runtime API field "media_io_kwargs". Ensure proper validation and error handling. @@ -91,14 +91,17 @@ class VideoMediaIO(MediaIO[tuple[npt.NDArray, dict[str, Any]]]): self.kwargs = kwargs self.video_loader = VIDEO_LOADER_REGISTRY.load(video_loader_backend) - def load_bytes(self, data: bytes) -> tuple[npt.NDArray, dict[str, Any]]: - return self.video_loader.load_bytes( + def load_bytes( + self, data: bytes + ) -> MediaWithBytes[tuple[npt.NDArray, dict[str, Any]]]: + video = self.video_loader.load_bytes( data, num_frames=self.num_frames, **self.kwargs ) + return MediaWithBytes(video, data) def load_base64( self, media_type: str, data: str - ) -> tuple[npt.NDArray, dict[str, Any]]: + ) -> MediaWithBytes[tuple[npt.NDArray, dict[str, Any]]]: if media_type.lower() == "video/jpeg": load_frame = partial( self.image_io.load_base64, @@ -160,11 +163,13 @@ class VideoMediaIO(MediaIO[tuple[npt.NDArray, dict[str, Any]]]): "frames_indices": frames_indices, "do_sample_frames": self.kwargs.get("do_sample_frames", False), } - return frames, metadata + return MediaWithBytes((frames, metadata), data.encode()) return self.load_bytes(pybase64.b64decode(data)) - def load_file(self, filepath: Path) -> tuple[npt.NDArray, dict[str, Any]]: + def load_file( + self, filepath: Path + ) -> MediaWithBytes[tuple[npt.NDArray, dict[str, Any]]]: with filepath.open("rb") as f: data = f.read() diff --git a/vllm/multimodal/parse.py b/vllm/multimodal/parse.py index 9d1e005da7..729a5bad18 100644 --- a/vllm/multimodal/parse.py +++ b/vllm/multimodal/parse.py @@ -366,6 +366,20 @@ class VideoProcessorItems(ProcessorBatchItems[HfVideoItem | None]): self.metadata = metadata + def _unwrap(self, item: Any) -> Any: + if isinstance(item, tuple): + frames, metadata = item + return super()._unwrap(frames), metadata + return super()._unwrap(item) + + def get_item_for_hash(self, index: int) -> Any: + item = self.data[index] + if isinstance(item, MediaWithBytes) and isinstance(self.metadata, list): + metadata = self.metadata[index] + if metadata is not None: + return item, metadata + return item + def get_num_frames(self, item_idx: int) -> int: video = self.get(item_idx) if video is None: @@ -552,7 +566,10 @@ class MultiModalDataParser: def _get_video_with_metadata( self, video: VideoItem, - ) -> tuple[np.ndarray, dict[str, Any] | None]: + ) -> tuple[np.ndarray | MediaWithBytes[np.ndarray], dict[str, Any] | None]: + if isinstance(video, MediaWithBytes): + new_video, metadata = self._get_video_with_metadata(video.media) + return MediaWithBytes(new_video, video.original_bytes), metadata if isinstance(video, tuple): return video if isinstance(video, list): @@ -653,7 +670,11 @@ class MultiModalDataParser: else: data_items = data # type: ignore[assignment] - new_videos = list[tuple[np.ndarray, dict[str, Any] | None]]() + new_videos = list[ + np.ndarray + | MediaWithBytes[np.ndarray] + | tuple[np.ndarray | MediaWithBytes[np.ndarray], dict[str, Any]] + ]() metadata_lst: list[dict[str, Any] | None] = [] for data_item in data_items: video, metadata = self._get_video_with_metadata(data_item) @@ -664,12 +685,9 @@ class MultiModalDataParser: "Please check your video input in `multi_modal_data`" ) new_videos.append((video, metadata)) - metadata_lst.append(metadata) else: new_videos.append(video) - - if not self.video_needs_metadata: - metadata = None + metadata_lst.append(metadata) return VideoProcessorItems(new_videos, metadata=metadata_lst) diff --git a/vllm/multimodal/utils.py b/vllm/multimodal/utils.py index 401e6e517b..eae274a81e 100644 --- a/vllm/multimodal/utils.py +++ b/vllm/multimodal/utils.py @@ -24,7 +24,13 @@ from .inputs import ( MultiModalKwargsItem, MultiModalSharedField, ) -from .media import AudioMediaIO, ImageMediaIO, MediaConnector, VideoMediaIO +from .media import ( + AudioMediaIO, + ImageMediaIO, + MediaConnector, + MediaWithBytes, + VideoMediaIO, +) if TYPE_CHECKING: import torch.types @@ -332,7 +338,7 @@ def fetch_image( def fetch_video( video_url: str, video_io_kwargs: dict[str, Any] | None = None, -) -> tuple[npt.NDArray, dict[str, Any]]: +) -> MediaWithBytes[tuple[npt.NDArray, dict[str, Any]]]: """ Args: video_url: URL of the video file to fetch.