From 396204230423b7cc6798300926b8fa30190d26a9 Mon Sep 17 00:00:00 2001 From: WillZZZy <93454470+WillZZZy@users.noreply.github.com> Date: Wed, 12 Aug 2026 00:06:57 -0400 Subject: [PATCH] [Bugfix][V1][Multimodal] Recover from P0/P1 processor cache drift (#46747) (#46747) Signed-off-by: William Zhao Signed-off-by: WillZZZy <93454470+WillZZZy@users.noreply.github.com> Co-authored-by: Cyrus Leung Co-authored-by: Claude --- rust/src/chat/tests/chat.rs | 20 +--- .../engine-core-client/src/protocol/output.rs | 23 ++-- .../engine-core-client/src/tests/client.rs | 13 +-- .../src/tests/python_compat.py | 1 + rust/src/llm/tests/generate.rs | 29 +---- rust/src/server/src/grpc/tests.rs | 12 +-- .../server/src/routes/http_client_tests.rs | 12 +-- rust/src/server/src/routes/tests.rs | 27 +---- tests/multimodal/test_cache.py | 101 ++++++++++++++++++ vllm/multimodal/cache.py | 54 +++++++++- vllm/v1/engine/__init__.py | 5 + vllm/v1/engine/async_llm.py | 12 +++ vllm/v1/engine/core.py | 40 +++++++ 13 files changed, 236 insertions(+), 113 deletions(-) diff --git a/rust/src/chat/tests/chat.rs b/rust/src/chat/tests/chat.rs index 2bbc39512b..fd32506c6e 100644 --- a/rust/src/chat/tests/chat.rs +++ b/rust/src/chat/tests/chat.rs @@ -46,18 +46,9 @@ fn request_output( EngineCoreOutput { request_id: request_id.to_string(), new_token_ids, - new_logprobs: None, - new_prompt_logprobs_tensors: None, - pooling_output: None, finish_reason, stop_reason, - events: None, - kv_transfer_params: None, - ec_transfer_params: None, - trace_headers: None, - prefill_stats: None, - routed_experts: None, - num_nans_in_logits: 0, + ..Default::default() } } @@ -74,16 +65,9 @@ fn request_output_with_logprobs( new_token_ids, new_logprobs: new_logprobs.map(MaybeWireLogprobs::Direct), new_prompt_logprobs_tensors: new_prompt_logprobs_tensors.map(MaybeWireLogprobs::Direct), - pooling_output: None, finish_reason, stop_reason, - events: None, - kv_transfer_params: None, - ec_transfer_params: None, - trace_headers: None, - prefill_stats: None, - routed_experts: None, - num_nans_in_logits: 0, + ..Default::default() } } diff --git a/rust/src/engine-core-client/src/protocol/output.rs b/rust/src/engine-core-client/src/protocol/output.rs index 78b9ff2daf..8919e5fa69 100644 --- a/rust/src/engine-core-client/src/protocol/output.rs +++ b/rust/src/engine-core-client/src/protocol/output.rs @@ -112,6 +112,17 @@ pub struct EngineCoreOutput { /// Number of NaNs seen in logits. Values above zero indicate corruption. #[serde(default)] pub num_nans_in_logits: u32, + /// Multi-modal hashes the engine could not find in its receiver cache, + /// because the frontend sent `data: None` for an item the engine had + /// already evicted. Non-empty makes the output retryable: a frontend that + /// keeps a metadata-only shadow of the engine cache drops these entries and + /// resends the request with the item data attached. + /// + /// TODO: always `None` here, since this client has no multi-modal processor + /// cache and always sends item data inline. Act on it once the Rust + /// frontend grows one. + #[serde(default)] + pub mm_cache_miss_hashes: Option>, } impl EngineCoreOutput { @@ -372,18 +383,9 @@ mod tests { outputs: vec![EngineCoreOutput { request_id: "req-1".to_string(), new_token_ids: vec![42], - new_logprobs: None, - new_prompt_logprobs_tensors: None, - pooling_output: None, finish_reason: Some(EngineCoreFinishReason::Length), stop_reason: Some(StopReason::Text("stop".to_string())), - events: None, - kv_transfer_params: None, - ec_transfer_params: None, - trace_headers: None, - prefill_stats: None, - routed_experts: None, - num_nans_in_logits: 0, + ..Default::default() }], finished_requests: Some(BTreeSet::from(["req-1".to_string()])), ..Default::default() @@ -437,6 +439,7 @@ mod tests { prefill_stats: None, routed_experts: None, num_nans_in_logits: 0, + mm_cache_miss_hashes: None, }, ], scheduler_stats: None, diff --git a/rust/src/engine-core-client/src/tests/client.rs b/rust/src/engine-core-client/src/tests/client.rs index 3268190785..ca29ba7c9b 100644 --- a/rust/src/engine-core-client/src/tests/client.rs +++ b/rust/src/engine-core-client/src/tests/client.rs @@ -223,18 +223,8 @@ fn request_output( EngineCoreOutput { request_id: request_id.to_string(), new_token_ids, - new_logprobs: None, - new_prompt_logprobs_tensors: None, - pooling_output: None, finish_reason, - stop_reason: None, - events: None, - kv_transfer_params: None, - ec_transfer_params: None, - trace_headers: None, - prefill_stats: None, - routed_experts: None, - num_nans_in_logits: 0, + ..Default::default() } } @@ -2607,6 +2597,7 @@ fn python_msgpack_fixtures_match_rust_encoding() { prefill_stats: None, routed_experts: None, num_nans_in_logits: 0, + mm_cache_miss_hashes: None, }, ], scheduler_stats: None, diff --git a/rust/src/engine-core-client/src/tests/python_compat.py b/rust/src/engine-core-client/src/tests/python_compat.py index 0d95dbba19..e306c35625 100755 --- a/rust/src/engine-core-client/src/tests/python_compat.py +++ b/rust/src/engine-core-client/src/tests/python_compat.py @@ -97,6 +97,7 @@ class EngineCoreOutput( prefill_stats: object | None = None routed_experts: object | None = None num_nans_in_logits: int = 0 + mm_cache_miss_hashes: list[str] | None = None class EngineCoreOutputs( diff --git a/rust/src/llm/tests/generate.rs b/rust/src/llm/tests/generate.rs index 5e5d599cd9..2ff3faa8d4 100644 --- a/rust/src/llm/tests/generate.rs +++ b/rust/src/llm/tests/generate.rs @@ -47,18 +47,9 @@ fn request_output_with_events( EngineCoreOutput { request_id: request_id.to_string(), new_token_ids, - new_logprobs: None, - new_prompt_logprobs_tensors: None, - pooling_output: None, finish_reason, - stop_reason: None, events, - kv_transfer_params: None, - ec_transfer_params: None, - trace_headers: None, - prefill_stats: None, - routed_experts: None, - num_nans_in_logits: 0, + ..Default::default() } } @@ -74,16 +65,8 @@ fn request_output_with_logprobs( new_token_ids, new_logprobs: new_logprobs.map(MaybeWireLogprobs::Direct), new_prompt_logprobs_tensors: prompt_logprobs.map(MaybeWireLogprobs::Direct), - pooling_output: None, finish_reason, - stop_reason: None, - events: None, - kv_transfer_params: None, - ec_transfer_params: None, - trace_headers: None, - prefill_stats: None, - routed_experts: None, - num_nans_in_logits: 0, + ..Default::default() } } @@ -101,16 +84,10 @@ fn request_output_with_logprobs_and_kv( new_token_ids, new_logprobs: new_logprobs.map(MaybeWireLogprobs::Direct), new_prompt_logprobs_tensors: prompt_logprobs.map(MaybeWireLogprobs::Direct), - pooling_output: None, finish_reason, - stop_reason: None, - events: None, kv_transfer_params, ec_transfer_params, - trace_headers: None, - prefill_stats: None, - routed_experts: None, - num_nans_in_logits: 0, + ..Default::default() } } diff --git a/rust/src/server/src/grpc/tests.rs b/rust/src/server/src/grpc/tests.rs index 274ac67fec..7eabd27014 100644 --- a/rust/src/server/src/grpc/tests.rs +++ b/rust/src/server/src/grpc/tests.rs @@ -114,18 +114,8 @@ fn request_output( EngineCoreOutput { request_id: request_id.to_string(), new_token_ids, - new_logprobs: None, - new_prompt_logprobs_tensors: None, - pooling_output: None, finish_reason, - stop_reason: None, - events: None, - kv_transfer_params: None, - ec_transfer_params: None, - trace_headers: None, - prefill_stats: None, - routed_experts: None, - num_nans_in_logits: 0, + ..Default::default() } } diff --git a/rust/src/server/src/routes/http_client_tests.rs b/rust/src/server/src/routes/http_client_tests.rs index 54fc4a04cd..216670b695 100644 --- a/rust/src/server/src/routes/http_client_tests.rs +++ b/rust/src/server/src/routes/http_client_tests.rs @@ -96,18 +96,8 @@ fn request_output( EngineCoreOutput { request_id: request_id.to_string(), new_token_ids, - new_logprobs: None, - new_prompt_logprobs_tensors: None, - pooling_output: None, finish_reason, - stop_reason: None, - events: None, - kv_transfer_params: None, - ec_transfer_params: None, - trace_headers: None, - prefill_stats: None, - routed_experts: None, - num_nans_in_logits: 0, + ..Default::default() } } diff --git a/rust/src/server/src/routes/tests.rs b/rust/src/server/src/routes/tests.rs index 92f1d44868..ac65ae269a 100644 --- a/rust/src/server/src/routes/tests.rs +++ b/rust/src/server/src/routes/tests.rs @@ -76,18 +76,9 @@ fn request_output_with_stop_reason( EngineCoreOutput { request_id: request_id.to_string(), new_token_ids, - new_logprobs: None, - new_prompt_logprobs_tensors: None, - pooling_output: None, finish_reason, stop_reason, - events: None, - kv_transfer_params: None, - ec_transfer_params: None, - trace_headers: None, - prefill_stats: None, - routed_experts: None, - num_nans_in_logits: 0, + ..Default::default() } } @@ -104,16 +95,9 @@ fn request_output_with_logprobs( new_token_ids, new_logprobs: new_logprobs.map(MaybeWireLogprobs::Direct), new_prompt_logprobs_tensors: new_prompt_logprobs_tensors.map(MaybeWireLogprobs::Direct), - pooling_output: None, finish_reason, stop_reason, - events: None, - kv_transfer_params: None, - ec_transfer_params: None, - trace_headers: None, - prefill_stats: None, - routed_experts: None, - num_nans_in_logits: 0, + ..Default::default() } } @@ -132,16 +116,11 @@ fn request_output_with_logprobs_and_kv( new_token_ids, new_logprobs: new_logprobs.map(MaybeWireLogprobs::Direct), new_prompt_logprobs_tensors: new_prompt_logprobs_tensors.map(MaybeWireLogprobs::Direct), - pooling_output: None, finish_reason, stop_reason, - events: None, kv_transfer_params, ec_transfer_params, - trace_headers: None, - prefill_stats: None, - routed_experts: None, - num_nans_in_logits: 0, + ..Default::default() } } diff --git a/tests/multimodal/test_cache.py b/tests/multimodal/test_cache.py index 816d218dd6..e5845b12b1 100644 --- a/tests/multimodal/test_cache.py +++ b/tests/multimodal/test_cache.py @@ -7,11 +7,13 @@ import pytest import torch from vllm.config import ModelConfig, ParallelConfig, VllmConfig +from vllm.config.multimodal import MultiModalConfig from vllm.multimodal import MULTIMODAL_REGISTRY from vllm.multimodal.cache import ( BaseMultiModalProcessorCache, BaseMultiModalReceiverCache, MultiModalCache, + MultiModalCacheMissError, MultiModalProcessorCacheInItem, MultiModalProcessorCacheItem, MultiModalProcessorCacheItemMetadata, @@ -232,6 +234,105 @@ def test_ipc_enable_disable_consistency(is_cached_calls_per_iter): ) +class _StubModelConfig: + """Minimal model-config stand-in: the cache classes only call + get_multimodal_config(), so this lets us construct them directly without a + real model, keeping this unit test free of any Hugging Face / network lookup. + """ + + def __init__(self, mm_processor_cache_gb: float) -> None: + self._mm_config = MultiModalConfig(mm_processor_cache_gb=mm_processor_cache_gb) + + def get_multimodal_config(self) -> MultiModalConfig: + return self._mm_config + + +def test_mm_cache_miss_raises_and_recovers(): + """A P0/P1 multimodal cache drift must be recoverable, not a hard crash. + + Reproduces the production failure: the P0 sender cache believes an item is + cached on P1 (so it sends ``data=None``), but the P1 receiver cache does not + have it. The receiver must raise ``MultiModalCacheMissError`` (carrying the + hash) instead of asserting; ``P0.invalidate()`` must drop the stale shadow + entry; and resending the item with data must repopulate both caches. + + Caches are built directly (not via the registry) so the test needs no real + model or network. + """ + model_config = _StubModelConfig(mm_processor_cache_gb=1) + p0 = MultiModalProcessorSenderCache(model_config) # type: ignore[arg-type] + p1 = MultiModalReceiverCache(model_config) # type: ignore[arg-type] + + mm_hash = "image_A" + item = MultiModalKwargsItem.dummy(nbytes=64) + + # Drift: the item reaches P1 with no data (P0 would have sent data=None on a + # stale HIT) but P1 does not have it. Must raise a typed, retryable error + # carrying the hash -- not assert. + with pytest.raises(MultiModalCacheMissError) as exc_info: + p1.get_and_update_item(None, mm_hash) + assert exc_info.value.mm_hashes == [mm_hash] + + # Give P0 a (now stale) shadow entry for the hash. + miss = p0.get_and_update_item((item, []), mm_hash) + assert miss[0] is item # MISS path returns the data to forward to P1 + assert p0.is_cached_item(mm_hash) + # On the next request P0 short-circuits to data=None -- the drift bug when + # P1 lacks the item. + hit = p0.get_and_update_item((item, []), mm_hash) + assert hit[0] is None + + # Recovery: drop the stale P0 entry so the client's resend-with-data takes + # the MISS path again and repopulates P1. + p0.invalidate(mm_hash) + assert not p0.is_cached_item(mm_hash) + + resent = p0.get_and_update_item((item, []), mm_hash) + assert resent[0] is item # MISS again -> data is resent + assert p1.get_and_update_item(item, mm_hash) == item # P1 now caches it + # A subsequent uuid-only request now succeeds on P1. + assert p1.get_and_update_item(None, mm_hash) == item + + +def test_mm_cache_miss_batches_all_drifted_hashes(): + """All hashes drifted within one request must surface in a single error. + + Otherwise a request with k drifted items needs k client retries (each resend + un-shadows only the one reported hash). get_and_update_features collects every + miss and raises once, so a single retry recovers the whole request -- while the + non-drifted items in the same request are still ingested. + """ + model_config = _StubModelConfig(mm_processor_cache_gb=1) + p1 = MultiModalReceiverCache(model_config) # type: ignore[arg-type] + item = MultiModalKwargsItem.dummy(nbytes=64) + + def _feature( + mm_hash: str, data: MultiModalKwargsItem | None + ) -> MultiModalFeatureSpec: + return MultiModalFeatureSpec( + data=data, + modality="image", + identifier=mm_hash, + mm_position=PlaceholderRange(offset=0, length=1), + mm_hash=mm_hash, + ) + + # A and C carry data; B, D, E arrive data=None (stale P0 shadow HITs) but P1 + # has none of them -- all three must be reported together, in order. + features = [ + _feature("A", item), + _feature("B", None), + _feature("C", item), + _feature("D", None), + _feature("E", None), + ] + with pytest.raises(MultiModalCacheMissError) as exc_info: + p1.get_and_update_features(features) + assert exc_info.value.mm_hashes == ["B", "D", "E"] + # The non-drifted items were still ingested into P1 before the raise. + assert "A" in p1._cache and "C" in p1._cache + + def _run_test_cache_eviction_lru( p0_cache: BaseMultiModalProcessorCache, p1_cache: BaseMultiModalReceiverCache, diff --git a/vllm/multimodal/cache.py b/vllm/multimodal/cache.py index 833b405ff6..54d86b57e5 100644 --- a/vllm/multimodal/cache.py +++ b/vllm/multimodal/cache.py @@ -39,6 +39,29 @@ if TYPE_CHECKING: logger = init_logger(__name__) +class MultiModalCacheMissError(RuntimeError): + """Raised by the P1 receiver cache when items are requested with no data and + are not cached. + + P0 (frontend) keeps a metadata-only shadow of P1 (engine) and sends + ``data=None`` on a shadow hit. The two caches are updated in different orders + across processes, so they can drift -- leaving P0 referencing items P1 has + evicted. Raising (instead of asserting) lets the engine return a retryable + response and have P0 drop the stale entries + (``BaseMultiModalProcessorCache.invalidate``) so the client resends the data. + Carries every drifted ``mm_hash`` in the request so P0 can drop them all in one + pass -- one retry then recovers the whole request, not one item per retry. + """ + + def __init__(self, mm_hashes: list[str]) -> None: + super().__init__( + f"Multi-modal items {mm_hashes} are not in the receiver (P1) cache and " + "no data was provided to recompute them (P0/P1 cache drift); the request " + "should be retried with the multi-modal data attached." + ) + self.mm_hashes = mm_hashes + + class MultiModalProcessorCacheItem: """ The data to store inside `MultiModalProcessorOnlyCache`. @@ -295,6 +318,13 @@ class BaseMultiModalProcessorCache( """ return [self.is_cached_item(mm_hash) for mm_hash in mm_hashes] + def invalidate(self, mm_hash: str) -> None: + """Drop ``mm_hash`` from this P0 shadow cache to recover from P0/P1 drift. + + No-op by default; shadow caches that can drift from P1 override this. + """ + pass + def close(self) -> None: """Close the underlying cache, if needed.""" pass @@ -433,6 +463,12 @@ class MultiModalProcessorSenderCache(BaseMultiModalProcessorCache): def make_stats(self, *, delta: bool = False) -> CacheInfo: return self._cache.stat(delta=delta) + @override + def invalidate(self, mm_hash: str) -> None: + # Drop our stale shadow entry so the next request for this hash re-sends the + # data and repopulates P1 (see MultiModalCacheMissError). + self._cache.pop(mm_hash, None) + class ShmObjectStoreSenderCache(BaseMultiModalProcessorCache): """ @@ -602,9 +638,19 @@ class BaseMultiModalReceiverCache( cache_key = feature.mm_hash or feature.identifier self.touch_receiver_cache_item(cache_key, feature.data) + missing_mm_hashes: list[str] = [] for feature in mm_features: cache_key = feature.mm_hash or feature.identifier - feature.data = self.get_and_update_item(feature.data, cache_key) + try: + feature.data = self.get_and_update_item(feature.data, cache_key) + except MultiModalCacheMissError as e: + # Collect every drifted hash in this request before raising, so the + # engine can have P0 invalidate them all at once -- otherwise a + # request with k drifted items needs k client retries (each resend + # only un-shadows the one reported hash). + missing_mm_hashes.extend(e.mm_hashes) + if missing_mm_hashes: + raise MultiModalCacheMissError(missing_mm_hashes) return mm_features @abstractmethod @@ -657,7 +703,11 @@ class MultiModalReceiverCache(BaseMultiModalReceiverCache): if (cached_item := self._cache.get(mm_hash)) is not None: return cached_item - assert mm_item is not None, f"Expected a cached item for {mm_hash=}" + # No data and not cached here: P0 sent data=None trusting its shadow, but + # the P0/P1 caches have drifted. Raise a retryable error (not assert) so the + # engine can have P0 drop the stale entry and the client resend the data. + if mm_item is None: + raise MultiModalCacheMissError([mm_hash]) self._cache[mm_hash] = mm_item return mm_item diff --git a/vllm/v1/engine/__init__.py b/vllm/v1/engine/__init__.py index 0197438e23..cfe3ba8c28 100644 --- a/vllm/v1/engine/__init__.py +++ b/vllm/v1/engine/__init__.py @@ -211,6 +211,11 @@ class EngineCoreOutput( # The number of NaNs in logits. # A value greater than 0 indicates that the output is corrupted. num_nans_in_logits: int = 0 + # Multi-modal hashes missing from the P1 receiver cache (P0/P1 drift; see + # `MultiModalCacheMissError`). Non-empty => retryable: the frontend drops these + # from its sender cache and the request is resent with the data. Appended last + # so `array_like` positional serialization stays backward compatible. + mm_cache_miss_hashes: list[str] | None = None @property def finished(self) -> bool: diff --git a/vllm/v1/engine/async_llm.py b/vllm/v1/engine/async_llm.py index 39500941e2..540ad23606 100644 --- a/vllm/v1/engine/async_llm.py +++ b/vllm/v1/engine/async_llm.py @@ -679,6 +679,8 @@ class AsyncLLM(EngineClient): self._logger_ref = [self.logger_manager] logger_ref = self._logger_ref renderer = self.renderer + # P0 multi-modal sender ("shadow") cache; None for text-only models. + mm_processor_cache = renderer.mm_processor_cache chunk_size = envs.VLLM_V1_OUTPUT_PROC_CHUNK_SIZE async def output_handler(): @@ -706,6 +708,16 @@ class AsyncLLM(EngineClient): # NOTE: RequestOutputs are pushed to their queues. assert not processed_outputs.request_outputs + # 2b) Recover from P0/P1 cache drift: the engine flags hashes + # it couldn't find (mm_cache_miss_hashes); drop them from the + # P0 shadow so the client's retry resends the data and + # repopulates P1. Hot-path no-op (field is None otherwise). + if mm_processor_cache is not None: + for eco in outputs_slice: + if eco.mm_cache_miss_hashes: + for mm_hash in eco.mm_cache_miss_hashes: + mm_processor_cache.invalidate(mm_hash) + # Allow other asyncio tasks to run between chunks if end < num_outputs: await asyncio.sleep(0) diff --git a/vllm/v1/engine/core.py b/vllm/v1/engine/core.py index 4e7f25aee5..956c800c7f 100644 --- a/vllm/v1/engine/core.py +++ b/vllm/v1/engine/core.py @@ -32,6 +32,7 @@ from vllm.logger import init_logger from vllm.logging_utils.dump_input import dump_engine_exception from vllm.lora.request import LoRARequest from vllm.multimodal import MULTIMODAL_REGISTRY +from vllm.multimodal.cache import MultiModalCacheMissError from vllm.tasks import POOLING_TASKS, SupportedTask from vllm.tracing import instrument, maybe_init_worker_tracer from vllm.transformers_utils.config import maybe_register_config_serialize_by_value @@ -1710,6 +1711,11 @@ class EngineCoreProc(EngineCore): req: EngineCoreRequest = add_request_decoder.decode(data_frames) try: request = self.preprocess_add_request(req) + except MultiModalCacheMissError as e: + # P0/P1 shadow drift -- return a retryable signal (P0 + # drops the stale entry, client resends with data). + self._handle_mm_cache_miss(req, e) + continue except Exception: self._handle_request_preproc_error(req) continue @@ -1803,6 +1809,40 @@ class EngineCoreProc(EngineCore): # Limit the number of buffers to reuse. reuse_buffers.append(buffer) + def _handle_mm_cache_miss( + self, request: EngineCoreRequest, err: MultiModalCacheMissError + ) -> None: + """Return a retryable response for a P0/P1 cache-drift miss. + + Surfaces every drifted hash via ``EngineCoreOutput.mm_cache_miss_hashes`` so + the frontend drops them from its sender cache and the client resends with + data (see ``MultiModalCacheMissError``). Logged at warning, not exception, + because it is expected and self-healing. + """ + logger.warning( + "Multi-modal cache miss for request %s (mm_hashes=%s): P0/P1 cache " + "drift; returning a retryable response so the items are resent with data.", + request.request_id, + err.mm_hashes, + ) + self.output_queue.put_nowait( + ( + request.client_index, + EngineCoreOutputs( + engine_index=self.engine_index, + finished_requests={request.request_id}, + outputs=[ + EngineCoreOutput( + request_id=request.request_id, + new_token_ids=[], + finish_reason=FinishReason.ERROR, + mm_cache_miss_hashes=err.mm_hashes, + ) + ], + ), + ) + ) + @staticmethod def _send_msg_tracking_payload( socket: zmq.Socket, buffers: Sequence[bytestr]