[Feat][MM Hashing] include media_io_kwargs in multi-modal hashes (#54241)

Signed-off-by: Jan Kwiecinski <jkwiecinski@nvidia.com>
This commit is contained in:
Jan Kwieciński
2026-09-01 13:17:01 +00:00
committed by GitHub
parent 55aa766dc8
commit 1f1f628859
6 changed files with 163 additions and 35 deletions
+22
View File
@@ -10,10 +10,13 @@ import pytest
from vllm.config import ModelConfig
from vllm.exceptions import VLLMValidationError
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.hasher import MultiModalHasher
from vllm.multimodal.parse import MultiModalDataParser
from vllm.multimodal.processing.context import (
InputProcessingContext,
overlay_modality_mm_kwargs,
)
from vllm.multimodal.processing.inputs import ProcessorInputs
from vllm.multimodal.processing.processor import (
BaseMultiModalProcessor,
PlaceholderFeaturesInfo,
@@ -1262,3 +1265,22 @@ def test_mm_processor_kwargs_merge_then_overlay_preserves_scoping():
assert overlay_modality_mm_kwargs(merged, "video")["size"] == size
assert "size" not in overlay_modality_mm_kwargs(merged, "image")
assert "size" not in overlay_modality_mm_kwargs(merged, None)
def test_processor_inputs_hashes_partial_uuids():
rng = np.random.RandomState(0)
images = [random_image(rng, min_wh=8, max_wh=9) for _ in range(2)]
inputs = ProcessorInputs(
prompt=[],
mm_data_items=MultiModalDataParser().parse_mm_data({"image": images}),
mm_uuid_items={"image": ["image-uuid", None]},
)
assert inputs.get_mm_hashes("test-model", "blake3") == {
"image": [
"image-uuid",
MultiModalHasher.hash_kwargs(
"blake3", model_id="test-model", image=images[1]
),
]
}
+92
View File
@@ -0,0 +1,92 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
from vllm.assets.video import VideoAsset
from vllm.config import ModelConfig, VllmConfig
from vllm.multimodal.utils import encode_video_url
from vllm.renderers.hf import HfRenderer
from vllm.renderers.params import ChatParams
from vllm.tokenizers.registry import cached_tokenizer_from_config
pytestmark = pytest.mark.cpu_test
video_data_url = encode_video_url(VideoAsset("baby_reading").np_ndarrays)
DEFAULT_MEDIA_IO_KWARGS = {"video": {"num_frames": 32}}
DEFAULT_MM_PROCESSOR_KWARGS = {"videos_kwargs": {"do_resize": True}}
def _build_video_renderer() -> HfRenderer:
model_config = ModelConfig(
model="llava-hf/llava-onevision-qwen2-0.5b-ov-hf",
max_model_len=4096,
mm_processor_cache_gb=4.0,
)
return HfRenderer(
VllmConfig(model_config=model_config),
cached_tokenizer_from_config(model_config),
)
def _get_message_hash(
renderer: HfRenderer,
messages: list[dict],
media_io_kwargs: dict | None = None,
mm_processor_kwargs: dict | None = None,
) -> str:
media_io_kwargs = media_io_kwargs or DEFAULT_MEDIA_IO_KWARGS
mm_processor_kwargs = mm_processor_kwargs or DEFAULT_MM_PROCESSOR_KWARGS
_, inputs = renderer.render_chat(
[messages],
ChatParams(
media_io_kwargs=media_io_kwargs,
mm_processor_kwargs=mm_processor_kwargs,
),
prompt_extras={"mm_processor_kwargs": mm_processor_kwargs},
)
return inputs[0]["mm_hashes"]["video"][0]
def _video_chat_messages(client_uuid: str):
return [
{
"role": "user",
"content": [
{
"type": "video_url",
"video_url": {"url": video_data_url},
"uuid": client_uuid,
},
{"type": "text", "text": "Describe the video."},
],
}
]
def test_mm_hash_includes_uuid_and_processing_kwargs():
renderer = _build_video_renderer()
messages = _video_chat_messages(video_data_url)
base_hash = _get_message_hash(renderer, messages)
media_io_hash = _get_message_hash(
renderer,
messages,
media_io_kwargs={"video": {"num_frames": 4}},
)
uuid_hash = _get_message_hash(
renderer,
_video_chat_messages("different-video-uuid"),
)
mm_processor_hash = _get_message_hash(
renderer,
messages,
mm_processor_kwargs={"videos_kwargs": {"do_resize": False}},
)
assert base_hash != media_io_hash
assert base_hash != uuid_hash
assert base_hash != mm_processor_hash