diff --git a/docs/configuration/optimization.md b/docs/configuration/optimization.md index efa0f8b904..18122ea2c8 100644 --- a/docs/configuration/optimization.md +++ b/docs/configuration/optimization.md @@ -376,6 +376,9 @@ processes. ### Configuration You can adjust the size of the cache by setting the value of `mm_processor_cache_gb` (default 4 GiB). +A processed item larger than this budget is served uncached (with a warning) +instead of failing engine startup; raise `mm_processor_cache_gb` if you want +those items cached. If you do not benefit much from the cache, you can disable both IPC and processor caching completely via `mm_processor_cache_gb=0`. diff --git a/tests/multimodal/test_cache.py b/tests/multimodal/test_cache.py index e5845b12b1..c20e554e57 100644 --- a/tests/multimodal/test_cache.py +++ b/tests/multimodal/test_cache.py @@ -17,6 +17,7 @@ from vllm.multimodal.cache import ( MultiModalProcessorCacheInItem, MultiModalProcessorCacheItem, MultiModalProcessorCacheItemMetadata, + MultiModalProcessorOnlyCache, MultiModalProcessorSenderCache, MultiModalReceiverCache, ShmObjectStoreReceiverCache, @@ -247,6 +248,34 @@ class _StubModelConfig: return self._mm_config +@pytest.mark.skip_global_cleanup +def test_oversized_item_is_served_uncached(): + """Items larger than the processor cache must not crash insert. + + cachetools.LRUCache raises ValueError("value too large") when a single + item exceeds maxsize. Engine profiling uses a max-size dummy item, so this + used to abort EngineCore startup (vllm-project/vllm#52835). Skip the + insert and serve the item uncached instead. + """ + # 1 KiB cache; a 4 KiB item cannot fit even if the cache is empty. + model_config = _StubModelConfig(mm_processor_cache_gb=1024 / GiB_bytes) + item = MultiModalKwargsItem.dummy(nbytes=4096) + small = MultiModalKwargsItem.dummy(nbytes=64) + + p0_only = MultiModalProcessorOnlyCache(model_config) # type: ignore[arg-type] + assert p0_only.get_and_update_item((item, []), "big")[0] is item + assert not p0_only.is_cached_item("big") + assert p0_only.get_and_update_item((small, []), "small")[0] is small + assert p0_only.is_cached_item("small") + + p0 = MultiModalProcessorSenderCache(model_config) # type: ignore[arg-type] + p1 = MultiModalReceiverCache(model_config) # type: ignore[arg-type] + assert p0.get_and_update_item((item, []), "big")[0] is item + assert not p0.is_cached_item("big") + assert p1.get_and_update_item(item, "big") is item + assert "big" not in p1._cache + + def test_mm_cache_miss_raises_and_recovers(): """A P0/P1 multimodal cache drift must be recoverable, not a hard crash. diff --git a/tests/utils_/test_cache.py b/tests/utils_/test_cache.py index e361006fd8..8d09b22415 100644 --- a/tests/utils_/test_cache.py +++ b/tests/utils_/test_cache.py @@ -1,5 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import pytest + from vllm.utils.cache import CacheInfo, LRUCache @@ -123,3 +125,16 @@ def test_lru_cache(): assert 2 in cache assert 4 in cache assert 6 in cache + + +def test_lru_cache_put_if_fits(): + cache = LRUCache(10, getsizeof=lambda x: x) + + assert cache.put_if_fits("ok", 4) is True + assert cache["ok"] == 4 + + assert cache.put_if_fits("big", 11) is False + assert "big" not in cache + + with pytest.raises(ValueError, match="value too large"): + cache.put("big", 11) diff --git a/vllm/config/multimodal.py b/vllm/config/multimodal.py index 04e94735a1..ce2217d9fe 100644 --- a/vllm/config/multimodal.py +++ b/vllm/config/multimodal.py @@ -157,6 +157,9 @@ class MultiModalConfig: resulting in a total memory usage of `mm_processor_cache_gb * (api_server_count + data_parallel_size)`. + A single processed item larger than this budget is served uncached + (with a warning) instead of failing. Raise this value to cache such items. + Set to `0` to disable this cache completely (not recommended).""" mm_processor_cache_type: MMCacheType = "lru" """Type of cache to use for the multi-modal preprocessor/mapper. If `shm`, diff --git a/vllm/multimodal/cache.py b/vllm/multimodal/cache.py index 54d86b57e5..5ec160d2f8 100644 --- a/vllm/multimodal/cache.py +++ b/vllm/multimodal/cache.py @@ -266,6 +266,42 @@ class BaseMultiModalCache(ABC, Generic[_I, _O]): for mm_item, mm_hash in zip(mm_items, mm_hashes) ] + def cache_if_fits( + self, + cache: LRUCache[str, _V], + key: str, + value: _V, + ) -> bool: + """Insert `value` if it fits in `cache`. + + `cachetools.Cache` raises `ValueError("value too large")` when a + single item exceeds `maxsize`. An item bigger than the whole + processor cache can never be a hit, so skip the insert and serve it + uncached instead of aborting engine startup. + + LRU P0/P1 caches call this so they stay mirrored. SHM subclasses do + not use it; they already skip oversize items in `put()`. + + Args: + cache: The LRU cache to update. + key: Cache key (typically the multi-modal item hash). + value: Value to insert. + + Returns: + `True` if the item was cached, otherwise `False`. + """ + if cache.put_if_fits(key, value): + return True + logger.warning_once( + "Skipping multi-modal processor cache insert for an item of " + "%s GiB because it exceeds --mm-processor-cache-gb=%s. " + "The item will be processed uncached; increase " + "--mm-processor-cache-gb to cache items of this size.", + format_gib(int(cache.getsizeof(value))), + format_gib(int(cache.maxsize)), + ) + return False + @abstractmethod def clear_cache(self) -> None: """Clear the underlying cache.""" @@ -389,8 +425,7 @@ class MultiModalProcessorOnlyCache(BaseMultiModalProcessorCache): assert mm_item is not None, f"Expected a cached item for {mm_hash=}" - self._cache[mm_hash] = MultiModalProcessorCacheItem(*mm_item) - + self.cache_if_fits(self._cache, mm_hash, MultiModalProcessorCacheItem(*mm_item)) return mm_item @override @@ -447,8 +482,9 @@ class MultiModalProcessorSenderCache(BaseMultiModalProcessorCache): assert mm_item is not None, f"Expected a cached item for {mm_hash=}" - self._cache[mm_hash] = MultiModalProcessorCacheItemMetadata(*mm_item) - + self.cache_if_fits( + self._cache, mm_hash, MultiModalProcessorCacheItemMetadata(*mm_item) + ) return mm_item @override @@ -709,7 +745,7 @@ class MultiModalReceiverCache(BaseMultiModalReceiverCache): if mm_item is None: raise MultiModalCacheMissError([mm_hash]) - self._cache[mm_hash] = mm_item + self.cache_if_fits(self._cache, mm_hash, mm_item) return mm_item @override diff --git a/vllm/utils/cache.py b/vllm/utils/cache.py index e45c2f6a8b..791b8d848d 100644 --- a/vllm/utils/cache.py +++ b/vllm/utils/cache.py @@ -157,6 +157,23 @@ class LRUCache(cachetools.LRUCache[_K, _V]): def put(self, key: _K, value: _V) -> None: self.__setitem__(key, value) + def put_if_fits(self, key: _K, value: _V) -> bool: + """Insert `value` if it is not larger than the cache capacity. + + Unlike `put`, this does not raise when a single item exceeds + `maxsize`. Size is computed once inside the insert path. + + Returns: + `True` if the item was cached, otherwise `False`. + """ + try: + self[key] = value + except ValueError as e: + if str(e) != "value too large": + raise + return False + return True + def pin(self, key: _K) -> None: """ Pins a key in the cache preventing it from being