diff --git a/tests/multimodal/test_cache.py b/tests/multimodal/test_cache.py index 02bd511970..37ec5b0cb8 100644 --- a/tests/multimodal/test_cache.py +++ b/tests/multimodal/test_cache.py @@ -330,6 +330,51 @@ def test_mm_cache_miss_raises_and_recovers(): assert p1.get_and_update_item(None, mm_hash) == item +def test_receiver_cache_replaces_stale_item_when_payload_resent(): + """A P0 miss can resend a different item under the same identity. + + Independent LRU eviction leaves P1 holding the old tensor. Substituting it + for the new payload pairs the request's placeholders with the wrong item + and kills EngineCore. Prefer the fresh payload and keep it for later hits. + """ + model_config = _StubModelConfig(mm_processor_cache_gb=1) + p1 = MultiModalReceiverCache(model_config) # type: ignore[arg-type] + small = MultiModalKwargsItem.dummy(nbytes=64) + large = MultiModalKwargsItem.dummy(nbytes=256) + mm_hash = "shared-id" + + assert p1.get_and_update_item(small, mm_hash) is small + assert p1.get_and_update_item(large, mm_hash) is large + assert p1.get_and_update_item(None, mm_hash) is large + + +def test_receiver_cache_features_keep_resent_payload(): + """EngineCore updates features through get_and_update_features.""" + model_config = _StubModelConfig(mm_processor_cache_gb=1) + p1 = MultiModalReceiverCache(model_config) # type: ignore[arg-type] + small = MultiModalKwargsItem.dummy(nbytes=64) + large = MultiModalKwargsItem.dummy(nbytes=256) + mm_hash = "shared-id" + + def _feature( + data: MultiModalKwargsItem | None, length: int + ) -> MultiModalFeatureSpec: + return MultiModalFeatureSpec( + data=data, + modality="image", + identifier=mm_hash, + mm_position=PlaceholderRange(offset=0, length=length), + mm_hash=mm_hash, + ) + + seeded = p1.get_and_update_features([_feature(small, length=4)]) + assert seeded[0].data is small + + updated = p1.get_and_update_features([_feature(large, length=2048)]) + assert updated[0].data is large + assert updated[0].mm_position.length == 2048 + + def test_mm_cache_miss_batches_all_drifted_hashes(): """All hashes drifted within one request must surface in a single error. diff --git a/vllm/multimodal/cache.py b/vllm/multimodal/cache.py index b4fabfc21f..15d433112c 100644 --- a/vllm/multimodal/cache.py +++ b/vllm/multimodal/cache.py @@ -710,9 +710,12 @@ class MultiModalReceiverCache(BaseMultiModalReceiverCache): How to update each item: - - If the item is in the cache, replace the input with the cached item. - - If the item is not in the cache, store that item (which includes tensor - data) into the cache, and return the input. + - If the caller sent tensor data, store it (replacing any cached item + under the same key) and return that data. P0 can miss after independent + LRU eviction and resend a different item for the same identity. + - If the caller sent no data and the item is cached, return the cached item. + - If the caller sent no data and the item is not cached, raise + ``MultiModalCacheMissError``. """ def __init__(self, model_config: "ModelConfig") -> None: @@ -731,17 +734,22 @@ class MultiModalReceiverCache(BaseMultiModalReceiverCache): mm_item: MultiModalKwargsItem | None, mm_hash: str, ) -> MultiModalKwargsItem: + if mm_item is not None: + # P0 sent a payload. Never keep a stale cached tensor under this + # identity: a later P0 hit would pair new placeholders with the old + # item and crash the engine. Drop first so a too-large replacement + # cannot leave the previous entry in place. + self._cache.pop(mm_hash, None) + self.cache_if_fits(self._cache, mm_hash, mm_item) + return mm_item + if (cached_item := self._cache.get(mm_hash)) is not None: return cached_item # No data and not cached here: P0 sent data=None trusting its shadow, but # the P0/P1 caches have drifted. Raise a retryable error (not assert) so the # engine can have P0 drop the stale entry and the client resend the data. - if mm_item is None: - raise MultiModalCacheMissError([mm_hash]) - - self.cache_if_fits(self._cache, mm_hash, mm_item) - return mm_item + raise MultiModalCacheMissError([mm_hash]) @override def touch_receiver_cache_item(