[Bugfix] Missing cached item in the MultiModalReceiverCache (#28525)
Signed-off-by: knlnguyen1802 <knlnguyen1802@gmail.com> Co-authored-by: Chenguang Zheng <645327136@qq.com> Co-authored-by: Cyrus Leung <cyrus.tl.leung@gmail.com> Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
This commit is contained in:
co-authored by
Chenguang Zheng
Cyrus Leung
Cyrus Leung
parent
d0985c5feb
commit
fc6acc88ca
@@ -574,7 +574,6 @@ class SingleWriterShmObjectStorage:
|
||||
value
|
||||
)
|
||||
buffer_size = self.flag_bytes + data_bytes + md_bytes
|
||||
|
||||
# Sanity checks
|
||||
if buffer_size > self.max_object_size:
|
||||
raise ValueError(
|
||||
@@ -626,6 +625,44 @@ class SingleWriterShmObjectStorage:
|
||||
|
||||
return obj
|
||||
|
||||
def touch(
|
||||
self,
|
||||
key: str,
|
||||
address: int = 0,
|
||||
monotonic_id: int = 0,
|
||||
) -> None:
|
||||
"""
|
||||
Touch an existing cached item to update its eviction status.
|
||||
|
||||
For writers (ShmObjectStoreSenderCache): Increment writer_flag
|
||||
For readers (ShmObjectStoreReceiverCache): Increment reader_count
|
||||
|
||||
Args:
|
||||
key: String key of the object to touch
|
||||
address: Address of the object (only for readers)
|
||||
monotonic_id: Monotonic ID of the object (only for readers)
|
||||
|
||||
"""
|
||||
if self._reader_lock is None:
|
||||
if key not in self.key_index:
|
||||
return None
|
||||
address, monotonic_id = self.key_index[key]
|
||||
# Writer side: increment writer_flag to raise eviction threshold
|
||||
self.increment_writer_flag(monotonic_id)
|
||||
else:
|
||||
with (
|
||||
self._reader_lock,
|
||||
self.ring_buffer.access_buf(address) as (data_view, _),
|
||||
):
|
||||
reader_count = self.ring_buffer.byte2int(data_view[: self.flag_bytes])
|
||||
|
||||
# NOTE(Long):
|
||||
# Avoid increasing flag on newly added item (sync with sender)
|
||||
# Since when a new item is added
|
||||
# pre-touch has no effect on writer side
|
||||
if reader_count >= self.n_readers:
|
||||
self.increment_reader_flag(data_view[: self.flag_bytes])
|
||||
|
||||
def handle(self):
|
||||
"""Get handle for sharing across processes."""
|
||||
return ShmObjectStorageHandle(
|
||||
|
||||
@@ -302,6 +302,19 @@ class BaseMultiModalProcessorCache(
|
||||
"""
|
||||
return [self.is_cached_item(mm_hash) for mm_hash in mm_hashes]
|
||||
|
||||
@abstractmethod
|
||||
def touch_sender_cache_item(self, mm_hash: str) -> None:
|
||||
"""
|
||||
Update the cache eviction order for a multi-modal item.
|
||||
|
||||
This is used to touch the item in the cache without changing
|
||||
its value.
|
||||
|
||||
Args:
|
||||
mm_hash: The hash of the multi-modal item.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
def make_stats(self, *, delta: bool = False) -> CacheInfo:
|
||||
"""
|
||||
@@ -353,6 +366,10 @@ class MultiModalProcessorOnlyCache(BaseMultiModalProcessorCache):
|
||||
|
||||
return mm_item
|
||||
|
||||
@override
|
||||
def touch_sender_cache_item(self, mm_hash: str) -> None:
|
||||
self._cache.touch(mm_hash)
|
||||
|
||||
@override
|
||||
def clear_cache(self) -> None:
|
||||
self._cache.clear()
|
||||
@@ -407,6 +424,10 @@ class MultiModalProcessorSenderCache(BaseMultiModalProcessorCache):
|
||||
|
||||
return mm_item
|
||||
|
||||
@override
|
||||
def touch_sender_cache_item(self, mm_hash: str) -> None:
|
||||
self._cache.touch(mm_hash)
|
||||
|
||||
@override
|
||||
def clear_cache(self) -> None:
|
||||
self._cache.clear()
|
||||
@@ -501,6 +522,12 @@ class ShmObjectStoreSenderCache(BaseMultiModalProcessorCache):
|
||||
logger.debug("Failed to cache mm_input with hash %s: %s", mm_hash, e)
|
||||
return mm_item
|
||||
|
||||
@override
|
||||
def touch_sender_cache_item(self, mm_hash: str) -> None:
|
||||
"""Touch the item in shared memory cache to prevent eviction.
|
||||
Increments writer_flag on sender side."""
|
||||
self._shm_cache.touch(mm_hash)
|
||||
|
||||
@override
|
||||
def clear_cache(self) -> None:
|
||||
self._shm_cache.clear()
|
||||
@@ -610,11 +637,37 @@ class BaseMultiModalReceiverCache(
|
||||
self,
|
||||
mm_features: list["MultiModalFeatureSpec"],
|
||||
) -> list["MultiModalFeatureSpec"]:
|
||||
"""Update multimodal features with cached encoder outputs."""
|
||||
"""
|
||||
Update multimodal features with cached encoder outputs.
|
||||
Touch all identifier at first before update to avoid
|
||||
item in updated list evict during update.
|
||||
"""
|
||||
for feature in mm_features:
|
||||
self.touch_receiver_cache_item(feature.identifier, feature.data)
|
||||
|
||||
for feature in mm_features:
|
||||
feature.data = self.get_and_update_item(feature.data, feature.identifier)
|
||||
return mm_features
|
||||
|
||||
@abstractmethod
|
||||
def touch_receiver_cache_item(
|
||||
self,
|
||||
mm_hash: str,
|
||||
mm_item: MultiModalKwargsItem | None = None,
|
||||
) -> None:
|
||||
"""
|
||||
Update the cache eviction order for a multi-modal item.
|
||||
|
||||
This is used to touch the item in the cache without changing
|
||||
its value.
|
||||
|
||||
Args:
|
||||
mm_hash: The hash of the multi-modal item.
|
||||
mm_item: The multi-modal item itself. This is optional and
|
||||
may not be needed by some cache implementations.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
class MultiModalReceiverCache(BaseMultiModalReceiverCache):
|
||||
"""
|
||||
@@ -651,6 +704,14 @@ class MultiModalReceiverCache(BaseMultiModalReceiverCache):
|
||||
self._cache[mm_hash] = mm_item
|
||||
return mm_item
|
||||
|
||||
@override
|
||||
def touch_receiver_cache_item(
|
||||
self,
|
||||
mm_hash: str,
|
||||
mm_item: MultiModalKwargsItem | None = None,
|
||||
) -> None:
|
||||
self._cache.touch(mm_hash)
|
||||
|
||||
@override
|
||||
def clear_cache(self) -> None:
|
||||
self._cache.clear()
|
||||
@@ -703,6 +764,20 @@ class ShmObjectStoreReceiverCache(BaseMultiModalReceiverCache):
|
||||
|
||||
return mm_item
|
||||
|
||||
@override
|
||||
def touch_receiver_cache_item(
|
||||
self,
|
||||
mm_hash: str,
|
||||
mm_item: MultiModalKwargsItem | None = None,
|
||||
) -> None:
|
||||
"""Touch the item in shared memory cache to prevent eviction.
|
||||
Increments reader_count on receiver side."""
|
||||
assert mm_item is not None
|
||||
if "address" in mm_item:
|
||||
address = cast(int, mm_item["address"].data)
|
||||
monotonic_id = cast(int, mm_item["monotonic_id"].data)
|
||||
self._shm_cache.touch(mm_hash, address=address, monotonic_id=monotonic_id)
|
||||
|
||||
@override
|
||||
def clear_cache(self) -> None:
|
||||
self._shm_cache.clear()
|
||||
|
||||
@@ -721,12 +721,12 @@ class MultiModalKwargsItem(UserDict[str, MultiModalFieldElem]):
|
||||
"""
|
||||
|
||||
@staticmethod
|
||||
def dummy(modality: str):
|
||||
def dummy(modality: str, nbytes: int = 1):
|
||||
"""Convenience class for testing."""
|
||||
mm_elem = MultiModalFieldElem(
|
||||
modality=modality,
|
||||
key="dummy",
|
||||
data=torch.empty(1),
|
||||
data=torch.empty(nbytes, dtype=torch.uint8),
|
||||
field=MultiModalSharedField(1),
|
||||
)
|
||||
return MultiModalKwargsItem.from_elems([mm_elem])
|
||||
|
||||
@@ -1248,7 +1248,13 @@ _I = TypeVar("_I", bound=BaseProcessingInfo)
|
||||
|
||||
MultiModalHashes = dict[str, list[str]]
|
||||
"""
|
||||
A collection of hashes with a similar structure as
|
||||
A collection of the multi-modal hash for each item, with a similar structure as
|
||||
[`MultiModalKwargsItems`][vllm.multimodal.inputs.MultiModalKwargsItems].
|
||||
"""
|
||||
|
||||
MultiModalIsCached = dict[str, list[bool]]
|
||||
"""
|
||||
A collection of the `is_cached` flag for each item, with a similar structure as
|
||||
[`MultiModalKwargsItems`][vllm.multimodal.inputs.MultiModalKwargsItems].
|
||||
"""
|
||||
|
||||
@@ -1725,7 +1731,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
cache: BaseMultiModalProcessorCache,
|
||||
mm_data_items: MultiModalDataItems,
|
||||
mm_hashes: MultiModalHashes,
|
||||
) -> MultiModalDataItems:
|
||||
) -> tuple[MultiModalIsCached, MultiModalDataItems]:
|
||||
mm_is_cached = {
|
||||
modality: cache.is_cached(hashes) for modality, hashes in mm_hashes.items()
|
||||
}
|
||||
@@ -1752,7 +1758,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
missing_modality_data.append(data)
|
||||
mm_missing_data[modality] = missing_modality_data
|
||||
|
||||
return self._to_mm_items(mm_missing_data)
|
||||
return mm_is_cached, self._to_mm_items(mm_missing_data)
|
||||
|
||||
def _recompute_cached_prompt_update(
|
||||
self,
|
||||
@@ -1769,14 +1775,15 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
self,
|
||||
cache: BaseMultiModalProcessorCache,
|
||||
mm_hashes: MultiModalHashes,
|
||||
mm_is_cached: MultiModalIsCached,
|
||||
mm_missing_kwargs: MultiModalKwargsItems,
|
||||
mm_missing_prompt_updates: MultiModalPromptUpdates,
|
||||
) -> tuple[MultiModalKwargsOptionalItems, MultiModalPromptUpdates]:
|
||||
# Need to calculate this at the beginning to avoid skipping cache logic
|
||||
# for subsequently repeated items in the same modality
|
||||
mm_is_cached = {
|
||||
modality: cache.is_cached(hashes) for modality, hashes in mm_hashes.items()
|
||||
}
|
||||
# Need to touch all mm hashes before update to avoid hash in updated
|
||||
# list evict during update
|
||||
for hashes in mm_hashes.values():
|
||||
for item_hash in hashes:
|
||||
cache.touch_sender_cache_item(item_hash)
|
||||
|
||||
mm_missing_next_idx = defaultdict[str, int](lambda: 0)
|
||||
|
||||
@@ -1789,15 +1796,14 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
missing_prompt_updates = mm_missing_prompt_updates.get(modality, [])
|
||||
|
||||
for item_idx, item_hash in enumerate(hashes):
|
||||
kwargs: MultiModalKwargsItem | None
|
||||
if not mm_is_cached[modality][item_idx]:
|
||||
missing_next_idx = mm_missing_next_idx[modality]
|
||||
kwargs = missing_kwargs[missing_next_idx]
|
||||
updates = missing_prompt_updates[missing_next_idx]
|
||||
missing_kwargs_item = missing_kwargs[missing_next_idx]
|
||||
missing_updates_item = missing_prompt_updates[missing_next_idx]
|
||||
|
||||
mm_missing_next_idx[modality] += 1
|
||||
|
||||
item = kwargs, updates
|
||||
item = missing_kwargs_item, missing_updates_item
|
||||
else:
|
||||
item = None
|
||||
|
||||
@@ -1896,7 +1902,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
mm_uuids=mm_uuids,
|
||||
)
|
||||
|
||||
mm_missing_data_items = self._get_cache_missing_items(
|
||||
mm_is_cached, mm_missing_data_items = self._get_cache_missing_items(
|
||||
cache=cache,
|
||||
mm_data_items=mm_data_items,
|
||||
mm_hashes=mm_hashes,
|
||||
@@ -1933,6 +1939,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
mm_kwargs, mm_prompt_updates = self._merge_mm_kwargs(
|
||||
cache,
|
||||
mm_hashes=mm_hashes,
|
||||
mm_is_cached=mm_is_cached,
|
||||
mm_missing_kwargs=mm_missing_kwargs,
|
||||
mm_missing_prompt_updates=mm_missing_prompt_updates,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user