From c6bca6e58540817bd6f192ae615dfa108cec1152 Mon Sep 17 00:00:00 2001 From: waizuichougou <105572299+waizuichougou@users.noreply.github.com> Date: Wed, 2 Sep 2026 22:20:26 +0800 Subject: [PATCH] [Bugfix][Multimodal] Scope cache hash kwargs by modality (#54918) Signed-off-by: waizuichougou <2082431897@qq.com> --- tests/multimodal/test_processing.py | 47 +++++++++++++++++++++++++ vllm/multimodal/processing/inputs.py | 51 ++++++++++++++++++++++------ 2 files changed, 88 insertions(+), 10 deletions(-) diff --git a/tests/multimodal/test_processing.py b/tests/multimodal/test_processing.py index 6e508af233..4a41013b21 100644 --- a/tests/multimodal/test_processing.py +++ b/tests/multimodal/test_processing.py @@ -1284,3 +1284,50 @@ def test_processor_inputs_hashes_partial_uuids(): ), ] } + + +def test_processor_inputs_hashes_scope_kwargs_by_modality(): + """Changing one modality's options must not invalidate another item.""" + rng = np.random.RandomState(0) + mm_data_items = MultiModalDataParser().parse_mm_data( + { + "image": [random_image(rng, min_wh=8, max_wh=9)], + "video": [np.zeros((2, 8, 8, 3), dtype=np.uint8)], + } + ) + mm_uuid_items = {"image": ["image-uuid"], "video": ["video-uuid"]} + + def get_hashes(video_frames: int, image_size: int, video_size: int): + return ProcessorInputs( + prompt=[], + mm_data_items=mm_data_items, + mm_uuid_items=mm_uuid_items, + media_io_kwargs={"video": {"num_frames": video_frames}}, + hf_processor_mm_kwargs={ + "images_kwargs": {"size": {"longest_edge": image_size}}, + "videos_kwargs": {"size": {"longest_edge": video_size}}, + }, + ).get_mm_hashes("test-model", "blake3") + + base = get_hashes(video_frames=4, image_size=224, video_size=224) + changed_video = get_hashes(video_frames=16, image_size=224, video_size=448) + changed_image = get_hashes(video_frames=4, image_size=448, video_size=224) + + assert changed_video["image"] == base["image"] + assert changed_video["video"] != base["video"] + assert changed_image["image"] != base["image"] + assert changed_image["video"] == base["video"] + + +def test_processor_inputs_hashes_ignore_unrelated_kwargs(): + """An image-only request ignores video-only processing configuration.""" + image = random_image(np.random.RandomState(0), min_wh=8, max_wh=9) + inputs = ProcessorInputs( + prompt=[], + mm_data_items=MultiModalDataParser().parse_mm_data({"image": [image]}), + mm_uuid_items={"image": ["image-uuid"]}, + media_io_kwargs={"video": {"num_frames": 16}}, + hf_processor_mm_kwargs={"videos_kwargs": {"size": {"longest_edge": 448}}}, + ) + + assert inputs.get_mm_hashes("test-model", "blake3") == {"image": ["image-uuid"]} diff --git a/vllm/multimodal/processing/inputs.py b/vllm/multimodal/processing/inputs.py index 9e9f71461b..56750c3087 100644 --- a/vllm/multimodal/processing/inputs.py +++ b/vllm/multimodal/processing/inputs.py @@ -9,6 +9,12 @@ from vllm.inputs import MultiModalHashes from ..hasher import MultiModalHasher from ..parse import MultiModalDataItems, MultiModalUUIDItems +_HF_MODALITY_PROCESSOR_KWARGS = { + "image": "images_kwargs", + "video": "videos_kwargs", + "audio": "audio_kwargs", +} + @dataclass class ProcessorInputs: @@ -31,17 +37,41 @@ class ProcessorInputs: mm_data_items = self.mm_data_items mm_uuid_items = self.mm_uuid_items or {} - hash_factors = { - "media_io_kwargs": dict(self.media_io_kwargs), - "mm_processor_kwargs": dict(self.hf_processor_mm_kwargs), - } - has_hash_factors = any(hash_factors.values()) - hash_factor_kwargs = hash_factors if has_hash_factors else {} - mm_hashes = dict[str, list[str]]() hasher = MultiModalHasher for modality, data_items in mm_data_items.items(): + # Each cache entry represents one item from one modality. Keep + # modality-scoped options out of unrelated entries, while + # retaining flat processor kwargs because they are shared by HF. + if modality in _HF_MODALITY_PROCESSOR_KWARGS: + modality_media_io_kwargs = self.media_io_kwargs.get(modality, {}) + media_io_kwargs = ( + {modality: modality_media_io_kwargs} + if modality_media_io_kwargs + else {} + ) + scoped_key = _HF_MODALITY_PROCESSOR_KWARGS[modality] + mm_processor_kwargs = { + key: value + for key, value in self.hf_processor_mm_kwargs.items() + if key not in _HF_MODALITY_PROCESSOR_KWARGS.values() + or key == scoped_key + } + else: + # Unified modalities such as ``vision_chunk`` can contain + # items originating from multiple media modalities. There is + # no per-item origin metadata here, so preserve all factors. + media_io_kwargs = dict(self.media_io_kwargs) + mm_processor_kwargs = dict(self.hf_processor_mm_kwargs) + + hash_factors = { + "media_io_kwargs": media_io_kwargs, + "mm_processor_kwargs": mm_processor_kwargs, + } + hash_factors = {key: value for key, value in hash_factors.items() if value} + has_hash_factors = bool(hash_factors) + uuid_items = ( mm_uuid_items[modality] if modality in mm_uuid_items @@ -53,8 +83,9 @@ class ProcessorInputs: for i, item in enumerate(data_items.get_all_items_for_hash()): uuid_item = uuid_items[i] - # NOTE: Even if a uuid_item is provided, model output - # depends on hash_factor_kwargs, so they are taken into account. + # NOTE: Even if a uuid_item is provided, model output depends + # on the current modality's hash factors, so they are taken + # into account. if uuid_item is None or has_hash_factors: # NOTE: use provided hash string to hash with kwargs # if available for better performance. @@ -64,7 +95,7 @@ class ProcessorInputs: hash_algorithm, model_id=model_id, **{modality: item}, - **hash_factor_kwargs, + **hash_factors, ) ) else: