From 1f1f628859dd0dc2a3b7f43810254884799619e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Kwieci=C5=84ski?= <153385549+Jankwi@users.noreply.github.com> Date: Tue, 1 Sep 2026 15:17:01 +0200 Subject: [PATCH] [Feat][MM Hashing] include media_io_kwargs in multi-modal hashes (#54241) Signed-off-by: Jan Kwiecinski --- tests/multimodal/test_processing.py | 22 ++++++ tests/renderers/test_multimodal_hashes.py | 92 +++++++++++++++++++++++ vllm/inputs/llm.py | 6 ++ vllm/multimodal/processing/inputs.py | 69 +++++++++-------- vllm/multimodal/processing/processor.py | 1 + vllm/renderers/base.py | 8 ++ 6 files changed, 163 insertions(+), 35 deletions(-) create mode 100644 tests/renderers/test_multimodal_hashes.py diff --git a/tests/multimodal/test_processing.py b/tests/multimodal/test_processing.py index 9d629fbf96..6e508af233 100644 --- a/tests/multimodal/test_processing.py +++ b/tests/multimodal/test_processing.py @@ -10,10 +10,13 @@ import pytest from vllm.config import ModelConfig from vllm.exceptions import VLLMValidationError from vllm.multimodal import MULTIMODAL_REGISTRY +from vllm.multimodal.hasher import MultiModalHasher +from vllm.multimodal.parse import MultiModalDataParser from vllm.multimodal.processing.context import ( InputProcessingContext, overlay_modality_mm_kwargs, ) +from vllm.multimodal.processing.inputs import ProcessorInputs from vllm.multimodal.processing.processor import ( BaseMultiModalProcessor, PlaceholderFeaturesInfo, @@ -1262,3 +1265,22 @@ def test_mm_processor_kwargs_merge_then_overlay_preserves_scoping(): assert overlay_modality_mm_kwargs(merged, "video")["size"] == size assert "size" not in overlay_modality_mm_kwargs(merged, "image") assert "size" not in overlay_modality_mm_kwargs(merged, None) + + +def test_processor_inputs_hashes_partial_uuids(): + rng = np.random.RandomState(0) + images = [random_image(rng, min_wh=8, max_wh=9) for _ in range(2)] + inputs = ProcessorInputs( + prompt=[], + mm_data_items=MultiModalDataParser().parse_mm_data({"image": images}), + mm_uuid_items={"image": ["image-uuid", None]}, + ) + + assert inputs.get_mm_hashes("test-model", "blake3") == { + "image": [ + "image-uuid", + MultiModalHasher.hash_kwargs( + "blake3", model_id="test-model", image=images[1] + ), + ] + } diff --git a/tests/renderers/test_multimodal_hashes.py b/tests/renderers/test_multimodal_hashes.py new file mode 100644 index 0000000000..a13c4a7d67 --- /dev/null +++ b/tests/renderers/test_multimodal_hashes.py @@ -0,0 +1,92 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import pytest + +from vllm.assets.video import VideoAsset +from vllm.config import ModelConfig, VllmConfig +from vllm.multimodal.utils import encode_video_url +from vllm.renderers.hf import HfRenderer +from vllm.renderers.params import ChatParams +from vllm.tokenizers.registry import cached_tokenizer_from_config + +pytestmark = pytest.mark.cpu_test + +video_data_url = encode_video_url(VideoAsset("baby_reading").np_ndarrays) + +DEFAULT_MEDIA_IO_KWARGS = {"video": {"num_frames": 32}} +DEFAULT_MM_PROCESSOR_KWARGS = {"videos_kwargs": {"do_resize": True}} + + +def _build_video_renderer() -> HfRenderer: + model_config = ModelConfig( + model="llava-hf/llava-onevision-qwen2-0.5b-ov-hf", + max_model_len=4096, + mm_processor_cache_gb=4.0, + ) + + return HfRenderer( + VllmConfig(model_config=model_config), + cached_tokenizer_from_config(model_config), + ) + + +def _get_message_hash( + renderer: HfRenderer, + messages: list[dict], + media_io_kwargs: dict | None = None, + mm_processor_kwargs: dict | None = None, +) -> str: + media_io_kwargs = media_io_kwargs or DEFAULT_MEDIA_IO_KWARGS + mm_processor_kwargs = mm_processor_kwargs or DEFAULT_MM_PROCESSOR_KWARGS + _, inputs = renderer.render_chat( + [messages], + ChatParams( + media_io_kwargs=media_io_kwargs, + mm_processor_kwargs=mm_processor_kwargs, + ), + prompt_extras={"mm_processor_kwargs": mm_processor_kwargs}, + ) + + return inputs[0]["mm_hashes"]["video"][0] + + +def _video_chat_messages(client_uuid: str): + return [ + { + "role": "user", + "content": [ + { + "type": "video_url", + "video_url": {"url": video_data_url}, + "uuid": client_uuid, + }, + {"type": "text", "text": "Describe the video."}, + ], + } + ] + + +def test_mm_hash_includes_uuid_and_processing_kwargs(): + renderer = _build_video_renderer() + messages = _video_chat_messages(video_data_url) + + base_hash = _get_message_hash(renderer, messages) + media_io_hash = _get_message_hash( + renderer, + messages, + media_io_kwargs={"video": {"num_frames": 4}}, + ) + uuid_hash = _get_message_hash( + renderer, + _video_chat_messages("different-video-uuid"), + ) + mm_processor_hash = _get_message_hash( + renderer, + messages, + mm_processor_kwargs={"videos_kwargs": {"do_resize": False}}, + ) + + assert base_hash != media_io_hash + assert base_hash != uuid_hash + assert base_hash != mm_processor_hash diff --git a/vllm/inputs/llm.py b/vllm/inputs/llm.py index f03661078c..a9ac8201fb 100644 --- a/vllm/inputs/llm.py +++ b/vllm/inputs/llm.py @@ -73,6 +73,12 @@ class _PromptOptions(TypedDict): if the model supports it. """ + media_io_kwargs: NotRequired[dict[str, dict[str, Any]] | None] + """ + Optional per-modality media loading and decoding arguments. + Used only for hash derivation when using multi-modal UUIDs. + """ + mm_processor_kwargs: NotRequired[dict[str, Any] | None] """ Optional multi-modal processor kwargs to be forwarded to the diff --git a/vllm/multimodal/processing/inputs.py b/vllm/multimodal/processing/inputs.py index ff6bd8b88b..9e9f71461b 100644 --- a/vllm/multimodal/processing/inputs.py +++ b/vllm/multimodal/processing/inputs.py @@ -21,6 +21,7 @@ class ProcessorInputs: mm_data_items: MultiModalDataItems mm_uuid_items: MultiModalUUIDItems | None = None hf_processor_mm_kwargs: Mapping[str, object] = field(default_factory=dict) + media_io_kwargs: Mapping[str, Mapping[str, object]] = field(default_factory=dict) def get_mm_hashes( self, @@ -29,49 +30,47 @@ class ProcessorInputs: ) -> MultiModalHashes: mm_data_items = self.mm_data_items mm_uuid_items = self.mm_uuid_items or {} - hf_processor_mm_kwargs = self.hf_processor_mm_kwargs + + hash_factors = { + "media_io_kwargs": dict(self.media_io_kwargs), + "mm_processor_kwargs": dict(self.hf_processor_mm_kwargs), + } + has_hash_factors = any(hash_factors.values()) + hash_factor_kwargs = hash_factors if has_hash_factors else {} mm_hashes = dict[str, list[str]]() hasher = MultiModalHasher for modality, data_items in mm_data_items.items(): - if modality in mm_uuid_items: - uuid_items = mm_uuid_items[modality] + uuid_items = ( + mm_uuid_items[modality] + if modality in mm_uuid_items + else ([None] * len(data_items)) + ) - # For None entries, compute a hash; otherwise, use provided ID. - hashes: list[str] = [] - for i, item in enumerate(data_items.get_all_items_for_hash()): - uuid_item = uuid_items[i] + # For None entries, compute a hash; otherwise, use provided ID. + hashes: list[str] = [] + for i, item in enumerate(data_items.get_all_items_for_hash()): + uuid_item = uuid_items[i] - # NOTE: Even if a uuid_item is provided, we still compute a hash - # if `hf_processor_mm_kwargs` is provided. - # This is because the processed multimodal inputs can be different - # depending on the processor kwargs. - if uuid_item is None or hf_processor_mm_kwargs: - # NOTE: use provided hash string to hash with kwargs - # if available for better performance. - item = uuid_item if uuid_item is not None else item - hashes.append( - hasher.hash_kwargs( - hash_algorithm, - model_id=model_id, - **{modality: item}, - **hf_processor_mm_kwargs, - ) + # NOTE: Even if a uuid_item is provided, model output + # depends on hash_factor_kwargs, so they are taken into account. + if uuid_item is None or has_hash_factors: + # NOTE: use provided hash string to hash with kwargs + # if available for better performance. + item = uuid_item if uuid_item is not None else item + hashes.append( + hasher.hash_kwargs( + hash_algorithm, + model_id=model_id, + **{modality: item}, + **hash_factor_kwargs, ) - else: - hashes.append(uuid_item) - - mm_hashes[modality] = hashes - else: - mm_hashes[modality] = [ - hasher.hash_kwargs( - hash_algorithm, - model_id=model_id, - **{modality: item}, - **hf_processor_mm_kwargs, ) - for item in data_items.get_all_items_for_hash() - ] + else: + # If there are no extra kwargs, use the client-provided UUID. + hashes.append(uuid_item) + + mm_hashes[modality] = hashes return mm_hashes diff --git a/vllm/multimodal/processing/processor.py b/vllm/multimodal/processing/processor.py index 593a48b337..afae854762 100644 --- a/vllm/multimodal/processing/processor.py +++ b/vllm/multimodal/processing/processor.py @@ -1811,6 +1811,7 @@ class EncDecMultiModalProcessor(BaseMultiModalProcessor[_I]): inputs.mm_data_items, inputs.mm_uuid_items, hf_processor_mm_kwargs=inputs.hf_processor_mm_kwargs, + media_io_kwargs=inputs.media_io_kwargs, ) encoder_inputs = super().apply(encoder_processor_inputs, timing_ctx) diff --git a/vllm/renderers/base.py b/vllm/renderers/base.py index b7eb7c41b5..540a400c7f 100644 --- a/vllm/renderers/base.py +++ b/vllm/renderers/base.py @@ -759,6 +759,7 @@ class BaseRenderer(ABC, Generic[_T]): mm_data: MultiModalDataDict, mm_uuids: MultiModalUUIDDict | None, mm_processor_kwargs: Mapping[str, object] | None, + media_io_kwargs: Mapping[str, Mapping[str, object]] | None = None, *, skip_mm_cache: bool = False, ) -> "MultiModalInput": @@ -781,6 +782,7 @@ class BaseRenderer(ABC, Generic[_T]): mm_data_items, mm_uuid_items, hf_processor_mm_kwargs=mm_processor_kwargs or {}, + media_io_kwargs=media_io_kwargs or {}, ) mm_timing_ctx = self._mm_timing_registry.get(mm_req_id) @@ -809,6 +811,7 @@ class BaseRenderer(ABC, Generic[_T]): multi_modal_data, mm_processor_kwargs=prompt.get("mm_processor_kwargs"), mm_uuids=prompt.get("multi_modal_uuids"), + media_io_kwargs=prompt.get("media_io_kwargs"), skip_mm_cache=skip_mm_cache, ) else: @@ -871,6 +874,7 @@ class BaseRenderer(ABC, Generic[_T]): multi_modal_data, mm_processor_kwargs=prompt.get("mm_processor_kwargs"), mm_uuids=prompt.get("multi_modal_uuids"), + media_io_kwargs=prompt.get("media_io_kwargs"), skip_mm_cache=skip_mm_cache, ) else: @@ -1082,6 +1086,8 @@ class BaseRenderer(ABC, Generic[_T]): tok_prompts = self.tokenize_prompts(dict_prompts, tok_params) + prompt_extras = dict(prompt_extras or {}) + prompt_extras["media_io_kwargs"] = chat_params.media_io_kwargs or {} self._apply_prompt_extras(tok_prompts, prompt_extras) eng_prompts = [ @@ -1118,6 +1124,8 @@ class BaseRenderer(ABC, Generic[_T]): tok_prompts = await self.tokenize_prompts_async(dict_prompts, tok_params) + prompt_extras = dict(prompt_extras or {}) + prompt_extras["media_io_kwargs"] = chat_params.media_io_kwargs or {} self._apply_prompt_extras(tok_prompts, prompt_extras) eng_prompts = await asyncio.gather(