[Bugfix] Missing cached item in the MultiModalReceiverCache (#28525)

Signed-off-by: knlnguyen1802 <knlnguyen1802@gmail.com>
Co-authored-by: Chenguang Zheng <645327136@qq.com>
Co-authored-by: Cyrus Leung <cyrus.tl.leung@gmail.com>
Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
This commit is contained in:
knlnguyen1802
2025-12-01 10:18:07 -08:00
committed by GitHub
co-authored by Chenguang Zheng Cyrus Leung Cyrus Leung
parent d0985c5feb
commit fc6acc88ca
5 changed files with 436 additions and 21 deletions
+20 -13
View File
@@ -1248,7 +1248,13 @@ _I = TypeVar("_I", bound=BaseProcessingInfo)
MultiModalHashes = dict[str, list[str]]
"""
A collection of hashes with a similar structure as
A collection of the multi-modal hash for each item, with a similar structure as
[`MultiModalKwargsItems`][vllm.multimodal.inputs.MultiModalKwargsItems].
"""
MultiModalIsCached = dict[str, list[bool]]
"""
A collection of the `is_cached` flag for each item, with a similar structure as
[`MultiModalKwargsItems`][vllm.multimodal.inputs.MultiModalKwargsItems].
"""
@@ -1725,7 +1731,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
cache: BaseMultiModalProcessorCache,
mm_data_items: MultiModalDataItems,
mm_hashes: MultiModalHashes,
) -> MultiModalDataItems:
) -> tuple[MultiModalIsCached, MultiModalDataItems]:
mm_is_cached = {
modality: cache.is_cached(hashes) for modality, hashes in mm_hashes.items()
}
@@ -1752,7 +1758,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
missing_modality_data.append(data)
mm_missing_data[modality] = missing_modality_data
return self._to_mm_items(mm_missing_data)
return mm_is_cached, self._to_mm_items(mm_missing_data)
def _recompute_cached_prompt_update(
self,
@@ -1769,14 +1775,15 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
self,
cache: BaseMultiModalProcessorCache,
mm_hashes: MultiModalHashes,
mm_is_cached: MultiModalIsCached,
mm_missing_kwargs: MultiModalKwargsItems,
mm_missing_prompt_updates: MultiModalPromptUpdates,
) -> tuple[MultiModalKwargsOptionalItems, MultiModalPromptUpdates]:
# Need to calculate this at the beginning to avoid skipping cache logic
# for subsequently repeated items in the same modality
mm_is_cached = {
modality: cache.is_cached(hashes) for modality, hashes in mm_hashes.items()
}
# Need to touch all mm hashes before update to avoid hash in updated
# list evict during update
for hashes in mm_hashes.values():
for item_hash in hashes:
cache.touch_sender_cache_item(item_hash)
mm_missing_next_idx = defaultdict[str, int](lambda: 0)
@@ -1789,15 +1796,14 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
missing_prompt_updates = mm_missing_prompt_updates.get(modality, [])
for item_idx, item_hash in enumerate(hashes):
kwargs: MultiModalKwargsItem | None
if not mm_is_cached[modality][item_idx]:
missing_next_idx = mm_missing_next_idx[modality]
kwargs = missing_kwargs[missing_next_idx]
updates = missing_prompt_updates[missing_next_idx]
missing_kwargs_item = missing_kwargs[missing_next_idx]
missing_updates_item = missing_prompt_updates[missing_next_idx]
mm_missing_next_idx[modality] += 1
item = kwargs, updates
item = missing_kwargs_item, missing_updates_item
else:
item = None
@@ -1896,7 +1902,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
mm_uuids=mm_uuids,
)
mm_missing_data_items = self._get_cache_missing_items(
mm_is_cached, mm_missing_data_items = self._get_cache_missing_items(
cache=cache,
mm_data_items=mm_data_items,
mm_hashes=mm_hashes,
@@ -1933,6 +1939,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
mm_kwargs, mm_prompt_updates = self._merge_mm_kwargs(
cache,
mm_hashes=mm_hashes,
mm_is_cached=mm_is_cached,
mm_missing_kwargs=mm_missing_kwargs,
mm_missing_prompt_updates=mm_missing_prompt_updates,
)