[Bugfix][V1][Multimodal] Recover from P0/P1 processor cache drift (#46747) (#46747)

Signed-off-by: William Zhao <williamzh@meta.com>
Signed-off-by: WillZZZy <93454470+WillZZZy@users.noreply.github.com>
Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
WillZZZy
2026-08-12 12:06:57 +08:00
committed by GitHub
co-authored by Cyrus Leung Claude
parent f4e9bb01de
commit 3962042304
13 changed files with 236 additions and 113 deletions
+101
View File
@@ -7,11 +7,13 @@ import pytest
import torch
from vllm.config import ModelConfig, ParallelConfig, VllmConfig
from vllm.config.multimodal import MultiModalConfig
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.cache import (
BaseMultiModalProcessorCache,
BaseMultiModalReceiverCache,
MultiModalCache,
MultiModalCacheMissError,
MultiModalProcessorCacheInItem,
MultiModalProcessorCacheItem,
MultiModalProcessorCacheItemMetadata,
@@ -232,6 +234,105 @@ def test_ipc_enable_disable_consistency(is_cached_calls_per_iter):
)
class _StubModelConfig:
"""Minimal model-config stand-in: the cache classes only call
get_multimodal_config(), so this lets us construct them directly without a
real model, keeping this unit test free of any Hugging Face / network lookup.
"""
def __init__(self, mm_processor_cache_gb: float) -> None:
self._mm_config = MultiModalConfig(mm_processor_cache_gb=mm_processor_cache_gb)
def get_multimodal_config(self) -> MultiModalConfig:
return self._mm_config
def test_mm_cache_miss_raises_and_recovers():
"""A P0/P1 multimodal cache drift must be recoverable, not a hard crash.
Reproduces the production failure: the P0 sender cache believes an item is
cached on P1 (so it sends ``data=None``), but the P1 receiver cache does not
have it. The receiver must raise ``MultiModalCacheMissError`` (carrying the
hash) instead of asserting; ``P0.invalidate()`` must drop the stale shadow
entry; and resending the item with data must repopulate both caches.
Caches are built directly (not via the registry) so the test needs no real
model or network.
"""
model_config = _StubModelConfig(mm_processor_cache_gb=1)
p0 = MultiModalProcessorSenderCache(model_config) # type: ignore[arg-type]
p1 = MultiModalReceiverCache(model_config) # type: ignore[arg-type]
mm_hash = "image_A"
item = MultiModalKwargsItem.dummy(nbytes=64)
# Drift: the item reaches P1 with no data (P0 would have sent data=None on a
# stale HIT) but P1 does not have it. Must raise a typed, retryable error
# carrying the hash -- not assert.
with pytest.raises(MultiModalCacheMissError) as exc_info:
p1.get_and_update_item(None, mm_hash)
assert exc_info.value.mm_hashes == [mm_hash]
# Give P0 a (now stale) shadow entry for the hash.
miss = p0.get_and_update_item((item, []), mm_hash)
assert miss[0] is item # MISS path returns the data to forward to P1
assert p0.is_cached_item(mm_hash)
# On the next request P0 short-circuits to data=None -- the drift bug when
# P1 lacks the item.
hit = p0.get_and_update_item((item, []), mm_hash)
assert hit[0] is None
# Recovery: drop the stale P0 entry so the client's resend-with-data takes
# the MISS path again and repopulates P1.
p0.invalidate(mm_hash)
assert not p0.is_cached_item(mm_hash)
resent = p0.get_and_update_item((item, []), mm_hash)
assert resent[0] is item # MISS again -> data is resent
assert p1.get_and_update_item(item, mm_hash) == item # P1 now caches it
# A subsequent uuid-only request now succeeds on P1.
assert p1.get_and_update_item(None, mm_hash) == item
def test_mm_cache_miss_batches_all_drifted_hashes():
"""All hashes drifted within one request must surface in a single error.
Otherwise a request with k drifted items needs k client retries (each resend
un-shadows only the one reported hash). get_and_update_features collects every
miss and raises once, so a single retry recovers the whole request -- while the
non-drifted items in the same request are still ingested.
"""
model_config = _StubModelConfig(mm_processor_cache_gb=1)
p1 = MultiModalReceiverCache(model_config) # type: ignore[arg-type]
item = MultiModalKwargsItem.dummy(nbytes=64)
def _feature(
mm_hash: str, data: MultiModalKwargsItem | None
) -> MultiModalFeatureSpec:
return MultiModalFeatureSpec(
data=data,
modality="image",
identifier=mm_hash,
mm_position=PlaceholderRange(offset=0, length=1),
mm_hash=mm_hash,
)
# A and C carry data; B, D, E arrive data=None (stale P0 shadow HITs) but P1
# has none of them -- all three must be reported together, in order.
features = [
_feature("A", item),
_feature("B", None),
_feature("C", item),
_feature("D", None),
_feature("E", None),
]
with pytest.raises(MultiModalCacheMissError) as exc_info:
p1.get_and_update_features(features)
assert exc_info.value.mm_hashes == ["B", "D", "E"]
# The non-drifted items were still ingested into P1 before the raise.
assert "A" in p1._cache and "C" in p1._cache
def _run_test_cache_eviction_lru(
p0_cache: BaseMultiModalProcessorCache,
p1_cache: BaseMultiModalReceiverCache,