[Bugfix] Skip MM processor cache inserts larger than capacity (#53016)

Signed-off-by: Prudhvivuda <prudhvi12042001@gmail.com>
This commit is contained in:
Prudhvi Vuda
2026-08-20 13:42:50 +00:00
committed by GitHub
parent 4b7cb949a9
commit de216b6e64
6 changed files with 108 additions and 5 deletions
+3
View File
@@ -376,6 +376,9 @@ processes.
### Configuration
You can adjust the size of the cache by setting the value of `mm_processor_cache_gb` (default 4 GiB).
A processed item larger than this budget is served uncached (with a warning)
instead of failing engine startup; raise `mm_processor_cache_gb` if you want
those items cached.
If you do not benefit much from the cache, you can disable both IPC
and processor caching completely via `mm_processor_cache_gb=0`.
+29
View File
@@ -17,6 +17,7 @@ from vllm.multimodal.cache import (
MultiModalProcessorCacheInItem,
MultiModalProcessorCacheItem,
MultiModalProcessorCacheItemMetadata,
MultiModalProcessorOnlyCache,
MultiModalProcessorSenderCache,
MultiModalReceiverCache,
ShmObjectStoreReceiverCache,
@@ -247,6 +248,34 @@ class _StubModelConfig:
return self._mm_config
@pytest.mark.skip_global_cleanup
def test_oversized_item_is_served_uncached():
"""Items larger than the processor cache must not crash insert.
cachetools.LRUCache raises ValueError("value too large") when a single
item exceeds maxsize. Engine profiling uses a max-size dummy item, so this
used to abort EngineCore startup (vllm-project/vllm#52835). Skip the
insert and serve the item uncached instead.
"""
# 1 KiB cache; a 4 KiB item cannot fit even if the cache is empty.
model_config = _StubModelConfig(mm_processor_cache_gb=1024 / GiB_bytes)
item = MultiModalKwargsItem.dummy(nbytes=4096)
small = MultiModalKwargsItem.dummy(nbytes=64)
p0_only = MultiModalProcessorOnlyCache(model_config) # type: ignore[arg-type]
assert p0_only.get_and_update_item((item, []), "big")[0] is item
assert not p0_only.is_cached_item("big")
assert p0_only.get_and_update_item((small, []), "small")[0] is small
assert p0_only.is_cached_item("small")
p0 = MultiModalProcessorSenderCache(model_config) # type: ignore[arg-type]
p1 = MultiModalReceiverCache(model_config) # type: ignore[arg-type]
assert p0.get_and_update_item((item, []), "big")[0] is item
assert not p0.is_cached_item("big")
assert p1.get_and_update_item(item, "big") is item
assert "big" not in p1._cache
def test_mm_cache_miss_raises_and_recovers():
"""A P0/P1 multimodal cache drift must be recoverable, not a hard crash.
+15
View File
@@ -1,5 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
from vllm.utils.cache import CacheInfo, LRUCache
@@ -123,3 +125,16 @@ def test_lru_cache():
assert 2 in cache
assert 4 in cache
assert 6 in cache
def test_lru_cache_put_if_fits():
cache = LRUCache(10, getsizeof=lambda x: x)
assert cache.put_if_fits("ok", 4) is True
assert cache["ok"] == 4
assert cache.put_if_fits("big", 11) is False
assert "big" not in cache
with pytest.raises(ValueError, match="value too large"):
cache.put("big", 11)
+3
View File
@@ -157,6 +157,9 @@ class MultiModalConfig:
resulting in a total memory usage of
`mm_processor_cache_gb * (api_server_count + data_parallel_size)`.
A single processed item larger than this budget is served uncached
(with a warning) instead of failing. Raise this value to cache such items.
Set to `0` to disable this cache completely (not recommended)."""
mm_processor_cache_type: MMCacheType = "lru"
"""Type of cache to use for the multi-modal preprocessor/mapper. If `shm`,
+41 -5
View File
@@ -266,6 +266,42 @@ class BaseMultiModalCache(ABC, Generic[_I, _O]):
for mm_item, mm_hash in zip(mm_items, mm_hashes)
]
def cache_if_fits(
self,
cache: LRUCache[str, _V],
key: str,
value: _V,
) -> bool:
"""Insert `value` if it fits in `cache`.
`cachetools.Cache` raises `ValueError("value too large")` when a
single item exceeds `maxsize`. An item bigger than the whole
processor cache can never be a hit, so skip the insert and serve it
uncached instead of aborting engine startup.
LRU P0/P1 caches call this so they stay mirrored. SHM subclasses do
not use it; they already skip oversize items in `put()`.
Args:
cache: The LRU cache to update.
key: Cache key (typically the multi-modal item hash).
value: Value to insert.
Returns:
`True` if the item was cached, otherwise `False`.
"""
if cache.put_if_fits(key, value):
return True
logger.warning_once(
"Skipping multi-modal processor cache insert for an item of "
"%s GiB because it exceeds --mm-processor-cache-gb=%s. "
"The item will be processed uncached; increase "
"--mm-processor-cache-gb to cache items of this size.",
format_gib(int(cache.getsizeof(value))),
format_gib(int(cache.maxsize)),
)
return False
@abstractmethod
def clear_cache(self) -> None:
"""Clear the underlying cache."""
@@ -389,8 +425,7 @@ class MultiModalProcessorOnlyCache(BaseMultiModalProcessorCache):
assert mm_item is not None, f"Expected a cached item for {mm_hash=}"
self._cache[mm_hash] = MultiModalProcessorCacheItem(*mm_item)
self.cache_if_fits(self._cache, mm_hash, MultiModalProcessorCacheItem(*mm_item))
return mm_item
@override
@@ -447,8 +482,9 @@ class MultiModalProcessorSenderCache(BaseMultiModalProcessorCache):
assert mm_item is not None, f"Expected a cached item for {mm_hash=}"
self._cache[mm_hash] = MultiModalProcessorCacheItemMetadata(*mm_item)
self.cache_if_fits(
self._cache, mm_hash, MultiModalProcessorCacheItemMetadata(*mm_item)
)
return mm_item
@override
@@ -709,7 +745,7 @@ class MultiModalReceiverCache(BaseMultiModalReceiverCache):
if mm_item is None:
raise MultiModalCacheMissError([mm_hash])
self._cache[mm_hash] = mm_item
self.cache_if_fits(self._cache, mm_hash, mm_item)
return mm_item
@override
+17
View File
@@ -157,6 +157,23 @@ class LRUCache(cachetools.LRUCache[_K, _V]):
def put(self, key: _K, value: _V) -> None:
self.__setitem__(key, value)
def put_if_fits(self, key: _K, value: _V) -> bool:
"""Insert `value` if it is not larger than the cache capacity.
Unlike `put`, this does not raise when a single item exceeds
`maxsize`. Size is computed once inside the insert path.
Returns:
`True` if the item was cached, otherwise `False`.
"""
try:
self[key] = value
except ValueError as e:
if str(e) != "value too large":
raise
return False
return True
def pin(self, key: _K) -> None:
"""
Pins a key in the cache preventing it from being