[Bugfix] Missing cached item in the MultiModalReceiverCache (#28525)
Signed-off-by: knlnguyen1802 <knlnguyen1802@gmail.com> Co-authored-by: Chenguang Zheng <645327136@qq.com> Co-authored-by: Cyrus Leung <cyrus.tl.leung@gmail.com> Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
This commit is contained in:
co-authored by
Chenguang Zheng
Cyrus Leung
Cyrus Leung
parent
d0985c5feb
commit
fc6acc88ca
@@ -1248,7 +1248,13 @@ _I = TypeVar("_I", bound=BaseProcessingInfo)
|
||||
|
||||
MultiModalHashes = dict[str, list[str]]
|
||||
"""
|
||||
A collection of hashes with a similar structure as
|
||||
A collection of the multi-modal hash for each item, with a similar structure as
|
||||
[`MultiModalKwargsItems`][vllm.multimodal.inputs.MultiModalKwargsItems].
|
||||
"""
|
||||
|
||||
MultiModalIsCached = dict[str, list[bool]]
|
||||
"""
|
||||
A collection of the `is_cached` flag for each item, with a similar structure as
|
||||
[`MultiModalKwargsItems`][vllm.multimodal.inputs.MultiModalKwargsItems].
|
||||
"""
|
||||
|
||||
@@ -1725,7 +1731,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
cache: BaseMultiModalProcessorCache,
|
||||
mm_data_items: MultiModalDataItems,
|
||||
mm_hashes: MultiModalHashes,
|
||||
) -> MultiModalDataItems:
|
||||
) -> tuple[MultiModalIsCached, MultiModalDataItems]:
|
||||
mm_is_cached = {
|
||||
modality: cache.is_cached(hashes) for modality, hashes in mm_hashes.items()
|
||||
}
|
||||
@@ -1752,7 +1758,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
missing_modality_data.append(data)
|
||||
mm_missing_data[modality] = missing_modality_data
|
||||
|
||||
return self._to_mm_items(mm_missing_data)
|
||||
return mm_is_cached, self._to_mm_items(mm_missing_data)
|
||||
|
||||
def _recompute_cached_prompt_update(
|
||||
self,
|
||||
@@ -1769,14 +1775,15 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
self,
|
||||
cache: BaseMultiModalProcessorCache,
|
||||
mm_hashes: MultiModalHashes,
|
||||
mm_is_cached: MultiModalIsCached,
|
||||
mm_missing_kwargs: MultiModalKwargsItems,
|
||||
mm_missing_prompt_updates: MultiModalPromptUpdates,
|
||||
) -> tuple[MultiModalKwargsOptionalItems, MultiModalPromptUpdates]:
|
||||
# Need to calculate this at the beginning to avoid skipping cache logic
|
||||
# for subsequently repeated items in the same modality
|
||||
mm_is_cached = {
|
||||
modality: cache.is_cached(hashes) for modality, hashes in mm_hashes.items()
|
||||
}
|
||||
# Need to touch all mm hashes before update to avoid hash in updated
|
||||
# list evict during update
|
||||
for hashes in mm_hashes.values():
|
||||
for item_hash in hashes:
|
||||
cache.touch_sender_cache_item(item_hash)
|
||||
|
||||
mm_missing_next_idx = defaultdict[str, int](lambda: 0)
|
||||
|
||||
@@ -1789,15 +1796,14 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
missing_prompt_updates = mm_missing_prompt_updates.get(modality, [])
|
||||
|
||||
for item_idx, item_hash in enumerate(hashes):
|
||||
kwargs: MultiModalKwargsItem | None
|
||||
if not mm_is_cached[modality][item_idx]:
|
||||
missing_next_idx = mm_missing_next_idx[modality]
|
||||
kwargs = missing_kwargs[missing_next_idx]
|
||||
updates = missing_prompt_updates[missing_next_idx]
|
||||
missing_kwargs_item = missing_kwargs[missing_next_idx]
|
||||
missing_updates_item = missing_prompt_updates[missing_next_idx]
|
||||
|
||||
mm_missing_next_idx[modality] += 1
|
||||
|
||||
item = kwargs, updates
|
||||
item = missing_kwargs_item, missing_updates_item
|
||||
else:
|
||||
item = None
|
||||
|
||||
@@ -1896,7 +1902,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
mm_uuids=mm_uuids,
|
||||
)
|
||||
|
||||
mm_missing_data_items = self._get_cache_missing_items(
|
||||
mm_is_cached, mm_missing_data_items = self._get_cache_missing_items(
|
||||
cache=cache,
|
||||
mm_data_items=mm_data_items,
|
||||
mm_hashes=mm_hashes,
|
||||
@@ -1933,6 +1939,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
mm_kwargs, mm_prompt_updates = self._merge_mm_kwargs(
|
||||
cache,
|
||||
mm_hashes=mm_hashes,
|
||||
mm_is_cached=mm_is_cached,
|
||||
mm_missing_kwargs=mm_missing_kwargs,
|
||||
mm_missing_prompt_updates=mm_missing_prompt_updates,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user