[Perf] Hash videos by source bytes (#49607)
Signed-off-by: Guan-Ming (Wesley) Chiu <105915352+guan404ming@users.noreply.github.com> Signed-off-by: Guan-Ming Chiu <105915352+guan404ming@users.noreply.github.com> Co-authored-by: Isotr0py <2037008807@qq.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
Isotr0py
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
d2ca3002d9
commit
04502deca2
@@ -9,6 +9,8 @@ import torch
|
||||
from PIL import Image, ImageDraw
|
||||
|
||||
from vllm.multimodal.hasher import MultiModalHasher
|
||||
from vllm.multimodal.media.base import MediaWithBytes
|
||||
from vllm.multimodal.parse import MultiModalDataParser
|
||||
|
||||
pytestmark = pytest.mark.cpu_test
|
||||
|
||||
@@ -82,6 +84,29 @@ def test_hash_collision_array_shape():
|
||||
assert hasher.hash_kwargs(data=arr1) != hasher.hash_kwargs(data=arr2)
|
||||
|
||||
|
||||
def test_hash_collision_video_num_frames():
|
||||
source = b"x" * 100
|
||||
|
||||
def item_for_hash(num_frames: int):
|
||||
frames: np.ndarray = np.zeros((num_frames, 8, 8, 3), dtype=np.uint8)
|
||||
metadata = {
|
||||
"total_num_frames": 16,
|
||||
"fps": 2.0,
|
||||
"duration": 8.0,
|
||||
"video_backend": "opencv",
|
||||
"frames_indices": list(range(num_frames)),
|
||||
"do_sample_frames": False,
|
||||
}
|
||||
video = MediaWithBytes((frames, metadata), source)
|
||||
items = MultiModalDataParser()._parse_video_data([video])
|
||||
return items.get_all_items_for_hash()[0]
|
||||
|
||||
hasher = MultiModalHasher
|
||||
assert hasher.hash_kwargs(video=item_for_hash(2)) != hasher.hash_kwargs(
|
||||
video=item_for_hash(4)
|
||||
)
|
||||
|
||||
|
||||
def test_hash_non_contiguous_array():
|
||||
arr = np.arange(24).reshape(4, 6).T
|
||||
assert not arr.flags.c_contiguous
|
||||
|
||||
@@ -83,6 +83,12 @@ class MultiModalHasher:
|
||||
|
||||
return cls.iter_item_to_bytes("image", obj.original_bytes)
|
||||
|
||||
if isinstance(obj, MediaWithBytes) and isinstance(obj.media, np.ndarray):
|
||||
frames = obj.media
|
||||
if frames.nbytes < len(obj.original_bytes):
|
||||
return cls.iter_item_to_bytes("video", frames)
|
||||
return cls.iter_item_to_bytes("video", obj.original_bytes)
|
||||
|
||||
if isinstance(obj, torch.Tensor):
|
||||
tensor_obj: torch.Tensor = obj.cpu()
|
||||
tensor_dtype = tensor_obj.dtype
|
||||
|
||||
@@ -66,7 +66,10 @@ these are directly passed to the model without HF processing.
|
||||
"""
|
||||
|
||||
VideoItem: TypeAlias = Union[
|
||||
HfVideoItem, "torch.Tensor", tuple[HfVideoItem, dict[str, Any]]
|
||||
HfVideoItem,
|
||||
"torch.Tensor",
|
||||
tuple[HfVideoItem, dict[str, Any]],
|
||||
MediaWithBytes[tuple[HfVideoItem, dict[str, Any]]],
|
||||
]
|
||||
"""
|
||||
A `transformers.video_utils.VideoInput` representing a single video item.
|
||||
|
||||
@@ -2,9 +2,10 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from collections.abc import Iterable, Iterator
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any, Generic, TypeVar
|
||||
from typing import Any, Generic, TypeVar, cast
|
||||
|
||||
import numpy as np
|
||||
|
||||
@@ -22,7 +23,8 @@ class MediaWithBytes(Generic[_T]):
|
||||
The wrapper delegates attribute access to the underlying media object,
|
||||
making it behave transparently like the wrapped type (e.g., PIL.Image).
|
||||
|
||||
NOTE: Currently, this wrapper is used only for the image modality.
|
||||
NOTE: Currently, this wrapper is used only for the image and video
|
||||
modalities.
|
||||
"""
|
||||
|
||||
media: _T
|
||||
@@ -32,6 +34,14 @@ class MediaWithBytes(Generic[_T]):
|
||||
"""Allow np.array(obj) to return np.array(obj.media)."""
|
||||
return np.array(self.media, *args, **kwargs)
|
||||
|
||||
def __iter__(self) -> Iterator[Any]:
|
||||
"""Allow unpacking obj to unpack obj.media (e.g. video tuples)."""
|
||||
return iter(cast(Iterable[Any], self.media))
|
||||
|
||||
def __getitem__(self, index: Any) -> Any:
|
||||
"""Allow obj[i] to index obj.media (e.g. video tuples)."""
|
||||
return cast(Any, self.media)[index]
|
||||
|
||||
def __getstate__(self):
|
||||
return self.__dict__.copy()
|
||||
|
||||
|
||||
@@ -29,7 +29,7 @@ from vllm.multimodal.video import get_video_loader_backend_for_processor
|
||||
from vllm.utils.registry import ExtensionManager
|
||||
|
||||
from .audio import AudioEmbeddingMediaIO, AudioMediaIO
|
||||
from .base import MediaIO
|
||||
from .base import MediaIO, MediaWithBytes
|
||||
from .image import ImageEmbeddingMediaIO, ImageMediaIO
|
||||
from .video import VideoMediaIO
|
||||
|
||||
@@ -536,7 +536,7 @@ class MediaConnector:
|
||||
*,
|
||||
image_mode: str | None = "RGB",
|
||||
video_processor: str | None = None,
|
||||
) -> tuple[npt.NDArray, dict[str, Any]]:
|
||||
) -> MediaWithBytes[tuple[npt.NDArray, dict[str, Any]]]:
|
||||
"""
|
||||
Load video from an HTTP or base64 data URL.
|
||||
"""
|
||||
@@ -562,7 +562,7 @@ class MediaConnector:
|
||||
*,
|
||||
image_mode: str | None = "RGB",
|
||||
video_processor: str | None = None,
|
||||
) -> tuple[npt.NDArray, dict[str, Any]]:
|
||||
) -> MediaWithBytes[tuple[npt.NDArray, dict[str, Any]]]:
|
||||
"""
|
||||
Asynchronously load video from an HTTP or base64 data URL.
|
||||
|
||||
|
||||
@@ -13,13 +13,13 @@ from vllm import envs
|
||||
from vllm.logger import init_logger
|
||||
|
||||
from ..video import VIDEO_LOADER_REGISTRY
|
||||
from .base import MediaIO
|
||||
from .base import MediaIO, MediaWithBytes
|
||||
from .image import ImageMediaIO
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class VideoMediaIO(MediaIO[tuple[npt.NDArray, dict[str, Any]]]):
|
||||
class VideoMediaIO(MediaIO[MediaWithBytes[tuple[npt.NDArray, dict[str, Any]]]]):
|
||||
"""Configuration values can be user-provided either by --media-io-kwargs or
|
||||
by the runtime API field "media_io_kwargs". Ensure proper validation and
|
||||
error handling.
|
||||
@@ -91,14 +91,17 @@ class VideoMediaIO(MediaIO[tuple[npt.NDArray, dict[str, Any]]]):
|
||||
self.kwargs = kwargs
|
||||
self.video_loader = VIDEO_LOADER_REGISTRY.load(video_loader_backend)
|
||||
|
||||
def load_bytes(self, data: bytes) -> tuple[npt.NDArray, dict[str, Any]]:
|
||||
return self.video_loader.load_bytes(
|
||||
def load_bytes(
|
||||
self, data: bytes
|
||||
) -> MediaWithBytes[tuple[npt.NDArray, dict[str, Any]]]:
|
||||
video = self.video_loader.load_bytes(
|
||||
data, num_frames=self.num_frames, **self.kwargs
|
||||
)
|
||||
return MediaWithBytes(video, data)
|
||||
|
||||
def load_base64(
|
||||
self, media_type: str, data: str
|
||||
) -> tuple[npt.NDArray, dict[str, Any]]:
|
||||
) -> MediaWithBytes[tuple[npt.NDArray, dict[str, Any]]]:
|
||||
if media_type.lower() == "video/jpeg":
|
||||
load_frame = partial(
|
||||
self.image_io.load_base64,
|
||||
@@ -160,11 +163,13 @@ class VideoMediaIO(MediaIO[tuple[npt.NDArray, dict[str, Any]]]):
|
||||
"frames_indices": frames_indices,
|
||||
"do_sample_frames": self.kwargs.get("do_sample_frames", False),
|
||||
}
|
||||
return frames, metadata
|
||||
return MediaWithBytes((frames, metadata), data.encode())
|
||||
|
||||
return self.load_bytes(pybase64.b64decode(data))
|
||||
|
||||
def load_file(self, filepath: Path) -> tuple[npt.NDArray, dict[str, Any]]:
|
||||
def load_file(
|
||||
self, filepath: Path
|
||||
) -> MediaWithBytes[tuple[npt.NDArray, dict[str, Any]]]:
|
||||
with filepath.open("rb") as f:
|
||||
data = f.read()
|
||||
|
||||
|
||||
@@ -366,6 +366,20 @@ class VideoProcessorItems(ProcessorBatchItems[HfVideoItem | None]):
|
||||
|
||||
self.metadata = metadata
|
||||
|
||||
def _unwrap(self, item: Any) -> Any:
|
||||
if isinstance(item, tuple):
|
||||
frames, metadata = item
|
||||
return super()._unwrap(frames), metadata
|
||||
return super()._unwrap(item)
|
||||
|
||||
def get_item_for_hash(self, index: int) -> Any:
|
||||
item = self.data[index]
|
||||
if isinstance(item, MediaWithBytes) and isinstance(self.metadata, list):
|
||||
metadata = self.metadata[index]
|
||||
if metadata is not None:
|
||||
return item, metadata
|
||||
return item
|
||||
|
||||
def get_num_frames(self, item_idx: int) -> int:
|
||||
video = self.get(item_idx)
|
||||
if video is None:
|
||||
@@ -552,7 +566,10 @@ class MultiModalDataParser:
|
||||
def _get_video_with_metadata(
|
||||
self,
|
||||
video: VideoItem,
|
||||
) -> tuple[np.ndarray, dict[str, Any] | None]:
|
||||
) -> tuple[np.ndarray | MediaWithBytes[np.ndarray], dict[str, Any] | None]:
|
||||
if isinstance(video, MediaWithBytes):
|
||||
new_video, metadata = self._get_video_with_metadata(video.media)
|
||||
return MediaWithBytes(new_video, video.original_bytes), metadata
|
||||
if isinstance(video, tuple):
|
||||
return video
|
||||
if isinstance(video, list):
|
||||
@@ -653,7 +670,11 @@ class MultiModalDataParser:
|
||||
else:
|
||||
data_items = data # type: ignore[assignment]
|
||||
|
||||
new_videos = list[tuple[np.ndarray, dict[str, Any] | None]]()
|
||||
new_videos = list[
|
||||
np.ndarray
|
||||
| MediaWithBytes[np.ndarray]
|
||||
| tuple[np.ndarray | MediaWithBytes[np.ndarray], dict[str, Any]]
|
||||
]()
|
||||
metadata_lst: list[dict[str, Any] | None] = []
|
||||
for data_item in data_items:
|
||||
video, metadata = self._get_video_with_metadata(data_item)
|
||||
@@ -664,12 +685,9 @@ class MultiModalDataParser:
|
||||
"Please check your video input in `multi_modal_data`"
|
||||
)
|
||||
new_videos.append((video, metadata))
|
||||
metadata_lst.append(metadata)
|
||||
else:
|
||||
new_videos.append(video)
|
||||
|
||||
if not self.video_needs_metadata:
|
||||
metadata = None
|
||||
metadata_lst.append(metadata)
|
||||
|
||||
return VideoProcessorItems(new_videos, metadata=metadata_lst)
|
||||
|
||||
|
||||
@@ -24,7 +24,13 @@ from .inputs import (
|
||||
MultiModalKwargsItem,
|
||||
MultiModalSharedField,
|
||||
)
|
||||
from .media import AudioMediaIO, ImageMediaIO, MediaConnector, VideoMediaIO
|
||||
from .media import (
|
||||
AudioMediaIO,
|
||||
ImageMediaIO,
|
||||
MediaConnector,
|
||||
MediaWithBytes,
|
||||
VideoMediaIO,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import torch.types
|
||||
@@ -332,7 +338,7 @@ def fetch_image(
|
||||
def fetch_video(
|
||||
video_url: str,
|
||||
video_io_kwargs: dict[str, Any] | None = None,
|
||||
) -> tuple[npt.NDArray, dict[str, Any]]:
|
||||
) -> MediaWithBytes[tuple[npt.NDArray, dict[str, Any]]]:
|
||||
"""
|
||||
Args:
|
||||
video_url: URL of the video file to fetch.
|
||||
|
||||
Reference in New Issue
Block a user