[Feat][MM Hashing] include media_io_kwargs in multi-modal hashes (#54241)
Signed-off-by: Jan Kwiecinski <jkwiecinski@nvidia.com>
This commit is contained in:
@@ -10,10 +10,13 @@ import pytest
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.hasher import MultiModalHasher
|
||||
from vllm.multimodal.parse import MultiModalDataParser
|
||||
from vllm.multimodal.processing.context import (
|
||||
InputProcessingContext,
|
||||
overlay_modality_mm_kwargs,
|
||||
)
|
||||
from vllm.multimodal.processing.inputs import ProcessorInputs
|
||||
from vllm.multimodal.processing.processor import (
|
||||
BaseMultiModalProcessor,
|
||||
PlaceholderFeaturesInfo,
|
||||
@@ -1262,3 +1265,22 @@ def test_mm_processor_kwargs_merge_then_overlay_preserves_scoping():
|
||||
assert overlay_modality_mm_kwargs(merged, "video")["size"] == size
|
||||
assert "size" not in overlay_modality_mm_kwargs(merged, "image")
|
||||
assert "size" not in overlay_modality_mm_kwargs(merged, None)
|
||||
|
||||
|
||||
def test_processor_inputs_hashes_partial_uuids():
|
||||
rng = np.random.RandomState(0)
|
||||
images = [random_image(rng, min_wh=8, max_wh=9) for _ in range(2)]
|
||||
inputs = ProcessorInputs(
|
||||
prompt=[],
|
||||
mm_data_items=MultiModalDataParser().parse_mm_data({"image": images}),
|
||||
mm_uuid_items={"image": ["image-uuid", None]},
|
||||
)
|
||||
|
||||
assert inputs.get_mm_hashes("test-model", "blake3") == {
|
||||
"image": [
|
||||
"image-uuid",
|
||||
MultiModalHasher.hash_kwargs(
|
||||
"blake3", model_id="test-model", image=images[1]
|
||||
),
|
||||
]
|
||||
}
|
||||
|
||||
@@ -0,0 +1,92 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.assets.video import VideoAsset
|
||||
from vllm.config import ModelConfig, VllmConfig
|
||||
from vllm.multimodal.utils import encode_video_url
|
||||
from vllm.renderers.hf import HfRenderer
|
||||
from vllm.renderers.params import ChatParams
|
||||
from vllm.tokenizers.registry import cached_tokenizer_from_config
|
||||
|
||||
pytestmark = pytest.mark.cpu_test
|
||||
|
||||
video_data_url = encode_video_url(VideoAsset("baby_reading").np_ndarrays)
|
||||
|
||||
DEFAULT_MEDIA_IO_KWARGS = {"video": {"num_frames": 32}}
|
||||
DEFAULT_MM_PROCESSOR_KWARGS = {"videos_kwargs": {"do_resize": True}}
|
||||
|
||||
|
||||
def _build_video_renderer() -> HfRenderer:
|
||||
model_config = ModelConfig(
|
||||
model="llava-hf/llava-onevision-qwen2-0.5b-ov-hf",
|
||||
max_model_len=4096,
|
||||
mm_processor_cache_gb=4.0,
|
||||
)
|
||||
|
||||
return HfRenderer(
|
||||
VllmConfig(model_config=model_config),
|
||||
cached_tokenizer_from_config(model_config),
|
||||
)
|
||||
|
||||
|
||||
def _get_message_hash(
|
||||
renderer: HfRenderer,
|
||||
messages: list[dict],
|
||||
media_io_kwargs: dict | None = None,
|
||||
mm_processor_kwargs: dict | None = None,
|
||||
) -> str:
|
||||
media_io_kwargs = media_io_kwargs or DEFAULT_MEDIA_IO_KWARGS
|
||||
mm_processor_kwargs = mm_processor_kwargs or DEFAULT_MM_PROCESSOR_KWARGS
|
||||
_, inputs = renderer.render_chat(
|
||||
[messages],
|
||||
ChatParams(
|
||||
media_io_kwargs=media_io_kwargs,
|
||||
mm_processor_kwargs=mm_processor_kwargs,
|
||||
),
|
||||
prompt_extras={"mm_processor_kwargs": mm_processor_kwargs},
|
||||
)
|
||||
|
||||
return inputs[0]["mm_hashes"]["video"][0]
|
||||
|
||||
|
||||
def _video_chat_messages(client_uuid: str):
|
||||
return [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{
|
||||
"type": "video_url",
|
||||
"video_url": {"url": video_data_url},
|
||||
"uuid": client_uuid,
|
||||
},
|
||||
{"type": "text", "text": "Describe the video."},
|
||||
],
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
def test_mm_hash_includes_uuid_and_processing_kwargs():
|
||||
renderer = _build_video_renderer()
|
||||
messages = _video_chat_messages(video_data_url)
|
||||
|
||||
base_hash = _get_message_hash(renderer, messages)
|
||||
media_io_hash = _get_message_hash(
|
||||
renderer,
|
||||
messages,
|
||||
media_io_kwargs={"video": {"num_frames": 4}},
|
||||
)
|
||||
uuid_hash = _get_message_hash(
|
||||
renderer,
|
||||
_video_chat_messages("different-video-uuid"),
|
||||
)
|
||||
mm_processor_hash = _get_message_hash(
|
||||
renderer,
|
||||
messages,
|
||||
mm_processor_kwargs={"videos_kwargs": {"do_resize": False}},
|
||||
)
|
||||
|
||||
assert base_hash != media_io_hash
|
||||
assert base_hash != uuid_hash
|
||||
assert base_hash != mm_processor_hash
|
||||
Reference in New Issue
Block a user