[Bugfix][Multimodal] Scope cache hash kwargs by modality (#54918)

Signed-off-by: waizuichougou <2082431897@qq.com>
This commit is contained in:
waizuichougou
2026-09-02 14:20:26 +00:00
committed by GitHub
parent 872084fb77
commit c6bca6e585
2 changed files with 88 additions and 10 deletions
+41 -10
View File
@@ -9,6 +9,12 @@ from vllm.inputs import MultiModalHashes
from ..hasher import MultiModalHasher
from ..parse import MultiModalDataItems, MultiModalUUIDItems
_HF_MODALITY_PROCESSOR_KWARGS = {
"image": "images_kwargs",
"video": "videos_kwargs",
"audio": "audio_kwargs",
}
@dataclass
class ProcessorInputs:
@@ -31,17 +37,41 @@ class ProcessorInputs:
mm_data_items = self.mm_data_items
mm_uuid_items = self.mm_uuid_items or {}
hash_factors = {
"media_io_kwargs": dict(self.media_io_kwargs),
"mm_processor_kwargs": dict(self.hf_processor_mm_kwargs),
}
has_hash_factors = any(hash_factors.values())
hash_factor_kwargs = hash_factors if has_hash_factors else {}
mm_hashes = dict[str, list[str]]()
hasher = MultiModalHasher
for modality, data_items in mm_data_items.items():
# Each cache entry represents one item from one modality. Keep
# modality-scoped options out of unrelated entries, while
# retaining flat processor kwargs because they are shared by HF.
if modality in _HF_MODALITY_PROCESSOR_KWARGS:
modality_media_io_kwargs = self.media_io_kwargs.get(modality, {})
media_io_kwargs = (
{modality: modality_media_io_kwargs}
if modality_media_io_kwargs
else {}
)
scoped_key = _HF_MODALITY_PROCESSOR_KWARGS[modality]
mm_processor_kwargs = {
key: value
for key, value in self.hf_processor_mm_kwargs.items()
if key not in _HF_MODALITY_PROCESSOR_KWARGS.values()
or key == scoped_key
}
else:
# Unified modalities such as ``vision_chunk`` can contain
# items originating from multiple media modalities. There is
# no per-item origin metadata here, so preserve all factors.
media_io_kwargs = dict(self.media_io_kwargs)
mm_processor_kwargs = dict(self.hf_processor_mm_kwargs)
hash_factors = {
"media_io_kwargs": media_io_kwargs,
"mm_processor_kwargs": mm_processor_kwargs,
}
hash_factors = {key: value for key, value in hash_factors.items() if value}
has_hash_factors = bool(hash_factors)
uuid_items = (
mm_uuid_items[modality]
if modality in mm_uuid_items
@@ -53,8 +83,9 @@ class ProcessorInputs:
for i, item in enumerate(data_items.get_all_items_for_hash()):
uuid_item = uuid_items[i]
# NOTE: Even if a uuid_item is provided, model output
# depends on hash_factor_kwargs, so they are taken into account.
# NOTE: Even if a uuid_item is provided, model output depends
# on the current modality's hash factors, so they are taken
# into account.
if uuid_item is None or has_hash_factors:
# NOTE: use provided hash string to hash with kwargs
# if available for better performance.
@@ -64,7 +95,7 @@ class ProcessorInputs:
hash_algorithm,
model_id=model_id,
**{modality: item},
**hash_factor_kwargs,
**hash_factors,
)
)
else: