[Bugfix] Skip MM processor cache inserts larger than capacity (#53016)
Signed-off-by: Prudhvivuda <prudhvi12042001@gmail.com>
This commit is contained in:
@@ -376,6 +376,9 @@ processes.
|
||||
### Configuration
|
||||
|
||||
You can adjust the size of the cache by setting the value of `mm_processor_cache_gb` (default 4 GiB).
|
||||
A processed item larger than this budget is served uncached (with a warning)
|
||||
instead of failing engine startup; raise `mm_processor_cache_gb` if you want
|
||||
those items cached.
|
||||
|
||||
If you do not benefit much from the cache, you can disable both IPC
|
||||
and processor caching completely via `mm_processor_cache_gb=0`.
|
||||
|
||||
@@ -17,6 +17,7 @@ from vllm.multimodal.cache import (
|
||||
MultiModalProcessorCacheInItem,
|
||||
MultiModalProcessorCacheItem,
|
||||
MultiModalProcessorCacheItemMetadata,
|
||||
MultiModalProcessorOnlyCache,
|
||||
MultiModalProcessorSenderCache,
|
||||
MultiModalReceiverCache,
|
||||
ShmObjectStoreReceiverCache,
|
||||
@@ -247,6 +248,34 @@ class _StubModelConfig:
|
||||
return self._mm_config
|
||||
|
||||
|
||||
@pytest.mark.skip_global_cleanup
|
||||
def test_oversized_item_is_served_uncached():
|
||||
"""Items larger than the processor cache must not crash insert.
|
||||
|
||||
cachetools.LRUCache raises ValueError("value too large") when a single
|
||||
item exceeds maxsize. Engine profiling uses a max-size dummy item, so this
|
||||
used to abort EngineCore startup (vllm-project/vllm#52835). Skip the
|
||||
insert and serve the item uncached instead.
|
||||
"""
|
||||
# 1 KiB cache; a 4 KiB item cannot fit even if the cache is empty.
|
||||
model_config = _StubModelConfig(mm_processor_cache_gb=1024 / GiB_bytes)
|
||||
item = MultiModalKwargsItem.dummy(nbytes=4096)
|
||||
small = MultiModalKwargsItem.dummy(nbytes=64)
|
||||
|
||||
p0_only = MultiModalProcessorOnlyCache(model_config) # type: ignore[arg-type]
|
||||
assert p0_only.get_and_update_item((item, []), "big")[0] is item
|
||||
assert not p0_only.is_cached_item("big")
|
||||
assert p0_only.get_and_update_item((small, []), "small")[0] is small
|
||||
assert p0_only.is_cached_item("small")
|
||||
|
||||
p0 = MultiModalProcessorSenderCache(model_config) # type: ignore[arg-type]
|
||||
p1 = MultiModalReceiverCache(model_config) # type: ignore[arg-type]
|
||||
assert p0.get_and_update_item((item, []), "big")[0] is item
|
||||
assert not p0.is_cached_item("big")
|
||||
assert p1.get_and_update_item(item, "big") is item
|
||||
assert "big" not in p1._cache
|
||||
|
||||
|
||||
def test_mm_cache_miss_raises_and_recovers():
|
||||
"""A P0/P1 multimodal cache drift must be recoverable, not a hard crash.
|
||||
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import pytest
|
||||
|
||||
from vllm.utils.cache import CacheInfo, LRUCache
|
||||
|
||||
|
||||
@@ -123,3 +125,16 @@ def test_lru_cache():
|
||||
assert 2 in cache
|
||||
assert 4 in cache
|
||||
assert 6 in cache
|
||||
|
||||
|
||||
def test_lru_cache_put_if_fits():
|
||||
cache = LRUCache(10, getsizeof=lambda x: x)
|
||||
|
||||
assert cache.put_if_fits("ok", 4) is True
|
||||
assert cache["ok"] == 4
|
||||
|
||||
assert cache.put_if_fits("big", 11) is False
|
||||
assert "big" not in cache
|
||||
|
||||
with pytest.raises(ValueError, match="value too large"):
|
||||
cache.put("big", 11)
|
||||
|
||||
@@ -157,6 +157,9 @@ class MultiModalConfig:
|
||||
resulting in a total memory usage of
|
||||
`mm_processor_cache_gb * (api_server_count + data_parallel_size)`.
|
||||
|
||||
A single processed item larger than this budget is served uncached
|
||||
(with a warning) instead of failing. Raise this value to cache such items.
|
||||
|
||||
Set to `0` to disable this cache completely (not recommended)."""
|
||||
mm_processor_cache_type: MMCacheType = "lru"
|
||||
"""Type of cache to use for the multi-modal preprocessor/mapper. If `shm`,
|
||||
|
||||
@@ -266,6 +266,42 @@ class BaseMultiModalCache(ABC, Generic[_I, _O]):
|
||||
for mm_item, mm_hash in zip(mm_items, mm_hashes)
|
||||
]
|
||||
|
||||
def cache_if_fits(
|
||||
self,
|
||||
cache: LRUCache[str, _V],
|
||||
key: str,
|
||||
value: _V,
|
||||
) -> bool:
|
||||
"""Insert `value` if it fits in `cache`.
|
||||
|
||||
`cachetools.Cache` raises `ValueError("value too large")` when a
|
||||
single item exceeds `maxsize`. An item bigger than the whole
|
||||
processor cache can never be a hit, so skip the insert and serve it
|
||||
uncached instead of aborting engine startup.
|
||||
|
||||
LRU P0/P1 caches call this so they stay mirrored. SHM subclasses do
|
||||
not use it; they already skip oversize items in `put()`.
|
||||
|
||||
Args:
|
||||
cache: The LRU cache to update.
|
||||
key: Cache key (typically the multi-modal item hash).
|
||||
value: Value to insert.
|
||||
|
||||
Returns:
|
||||
`True` if the item was cached, otherwise `False`.
|
||||
"""
|
||||
if cache.put_if_fits(key, value):
|
||||
return True
|
||||
logger.warning_once(
|
||||
"Skipping multi-modal processor cache insert for an item of "
|
||||
"%s GiB because it exceeds --mm-processor-cache-gb=%s. "
|
||||
"The item will be processed uncached; increase "
|
||||
"--mm-processor-cache-gb to cache items of this size.",
|
||||
format_gib(int(cache.getsizeof(value))),
|
||||
format_gib(int(cache.maxsize)),
|
||||
)
|
||||
return False
|
||||
|
||||
@abstractmethod
|
||||
def clear_cache(self) -> None:
|
||||
"""Clear the underlying cache."""
|
||||
@@ -389,8 +425,7 @@ class MultiModalProcessorOnlyCache(BaseMultiModalProcessorCache):
|
||||
|
||||
assert mm_item is not None, f"Expected a cached item for {mm_hash=}"
|
||||
|
||||
self._cache[mm_hash] = MultiModalProcessorCacheItem(*mm_item)
|
||||
|
||||
self.cache_if_fits(self._cache, mm_hash, MultiModalProcessorCacheItem(*mm_item))
|
||||
return mm_item
|
||||
|
||||
@override
|
||||
@@ -447,8 +482,9 @@ class MultiModalProcessorSenderCache(BaseMultiModalProcessorCache):
|
||||
|
||||
assert mm_item is not None, f"Expected a cached item for {mm_hash=}"
|
||||
|
||||
self._cache[mm_hash] = MultiModalProcessorCacheItemMetadata(*mm_item)
|
||||
|
||||
self.cache_if_fits(
|
||||
self._cache, mm_hash, MultiModalProcessorCacheItemMetadata(*mm_item)
|
||||
)
|
||||
return mm_item
|
||||
|
||||
@override
|
||||
@@ -709,7 +745,7 @@ class MultiModalReceiverCache(BaseMultiModalReceiverCache):
|
||||
if mm_item is None:
|
||||
raise MultiModalCacheMissError([mm_hash])
|
||||
|
||||
self._cache[mm_hash] = mm_item
|
||||
self.cache_if_fits(self._cache, mm_hash, mm_item)
|
||||
return mm_item
|
||||
|
||||
@override
|
||||
|
||||
@@ -157,6 +157,23 @@ class LRUCache(cachetools.LRUCache[_K, _V]):
|
||||
def put(self, key: _K, value: _V) -> None:
|
||||
self.__setitem__(key, value)
|
||||
|
||||
def put_if_fits(self, key: _K, value: _V) -> bool:
|
||||
"""Insert `value` if it is not larger than the cache capacity.
|
||||
|
||||
Unlike `put`, this does not raise when a single item exceeds
|
||||
`maxsize`. Size is computed once inside the insert path.
|
||||
|
||||
Returns:
|
||||
`True` if the item was cached, otherwise `False`.
|
||||
"""
|
||||
try:
|
||||
self[key] = value
|
||||
except ValueError as e:
|
||||
if str(e) != "value too large":
|
||||
raise
|
||||
return False
|
||||
return True
|
||||
|
||||
def pin(self, key: _K) -> None:
|
||||
"""
|
||||
Pins a key in the cache preventing it from being
|
||||
|
||||
Reference in New Issue
Block a user