[Bugfix][V1][Multimodal] Recover from P0/P1 processor cache drift (#46747) (#46747)

Signed-off-by: William Zhao <williamzh@meta.com>
Signed-off-by: WillZZZy <93454470+WillZZZy@users.noreply.github.com>
Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
WillZZZy
2026-08-12 12:06:57 +08:00
committed by GitHub
co-authored by Cyrus Leung Claude
parent f4e9bb01de
commit 3962042304
13 changed files with 236 additions and 113 deletions
+2 -18
View File
@@ -46,18 +46,9 @@ fn request_output(
EngineCoreOutput {
request_id: request_id.to_string(),
new_token_ids,
new_logprobs: None,
new_prompt_logprobs_tensors: None,
pooling_output: None,
finish_reason,
stop_reason,
events: None,
kv_transfer_params: None,
ec_transfer_params: None,
trace_headers: None,
prefill_stats: None,
routed_experts: None,
num_nans_in_logits: 0,
..Default::default()
}
}
@@ -74,16 +65,9 @@ fn request_output_with_logprobs(
new_token_ids,
new_logprobs: new_logprobs.map(MaybeWireLogprobs::Direct),
new_prompt_logprobs_tensors: new_prompt_logprobs_tensors.map(MaybeWireLogprobs::Direct),
pooling_output: None,
finish_reason,
stop_reason,
events: None,
kv_transfer_params: None,
ec_transfer_params: None,
trace_headers: None,
prefill_stats: None,
routed_experts: None,
num_nans_in_logits: 0,
..Default::default()
}
}
@@ -112,6 +112,17 @@ pub struct EngineCoreOutput {
/// Number of NaNs seen in logits. Values above zero indicate corruption.
#[serde(default)]
pub num_nans_in_logits: u32,
/// Multi-modal hashes the engine could not find in its receiver cache,
/// because the frontend sent `data: None` for an item the engine had
/// already evicted. Non-empty makes the output retryable: a frontend that
/// keeps a metadata-only shadow of the engine cache drops these entries and
/// resends the request with the item data attached.
///
/// TODO: always `None` here, since this client has no multi-modal processor
/// cache and always sends item data inline. Act on it once the Rust
/// frontend grows one.
#[serde(default)]
pub mm_cache_miss_hashes: Option<Vec<String>>,
}
impl EngineCoreOutput {
@@ -372,18 +383,9 @@ mod tests {
outputs: vec![EngineCoreOutput {
request_id: "req-1".to_string(),
new_token_ids: vec![42],
new_logprobs: None,
new_prompt_logprobs_tensors: None,
pooling_output: None,
finish_reason: Some(EngineCoreFinishReason::Length),
stop_reason: Some(StopReason::Text("stop".to_string())),
events: None,
kv_transfer_params: None,
ec_transfer_params: None,
trace_headers: None,
prefill_stats: None,
routed_experts: None,
num_nans_in_logits: 0,
..Default::default()
}],
finished_requests: Some(BTreeSet::from(["req-1".to_string()])),
..Default::default()
@@ -437,6 +439,7 @@ mod tests {
prefill_stats: None,
routed_experts: None,
num_nans_in_logits: 0,
mm_cache_miss_hashes: None,
},
],
scheduler_stats: None,
@@ -223,18 +223,8 @@ fn request_output(
EngineCoreOutput {
request_id: request_id.to_string(),
new_token_ids,
new_logprobs: None,
new_prompt_logprobs_tensors: None,
pooling_output: None,
finish_reason,
stop_reason: None,
events: None,
kv_transfer_params: None,
ec_transfer_params: None,
trace_headers: None,
prefill_stats: None,
routed_experts: None,
num_nans_in_logits: 0,
..Default::default()
}
}
@@ -2607,6 +2597,7 @@ fn python_msgpack_fixtures_match_rust_encoding() {
prefill_stats: None,
routed_experts: None,
num_nans_in_logits: 0,
mm_cache_miss_hashes: None,
},
],
scheduler_stats: None,
@@ -97,6 +97,7 @@ class EngineCoreOutput(
prefill_stats: object | None = None
routed_experts: object | None = None
num_nans_in_logits: int = 0
mm_cache_miss_hashes: list[str] | None = None
class EngineCoreOutputs(
+3 -26
View File
@@ -47,18 +47,9 @@ fn request_output_with_events(
EngineCoreOutput {
request_id: request_id.to_string(),
new_token_ids,
new_logprobs: None,
new_prompt_logprobs_tensors: None,
pooling_output: None,
finish_reason,
stop_reason: None,
events,
kv_transfer_params: None,
ec_transfer_params: None,
trace_headers: None,
prefill_stats: None,
routed_experts: None,
num_nans_in_logits: 0,
..Default::default()
}
}
@@ -74,16 +65,8 @@ fn request_output_with_logprobs(
new_token_ids,
new_logprobs: new_logprobs.map(MaybeWireLogprobs::Direct),
new_prompt_logprobs_tensors: prompt_logprobs.map(MaybeWireLogprobs::Direct),
pooling_output: None,
finish_reason,
stop_reason: None,
events: None,
kv_transfer_params: None,
ec_transfer_params: None,
trace_headers: None,
prefill_stats: None,
routed_experts: None,
num_nans_in_logits: 0,
..Default::default()
}
}
@@ -101,16 +84,10 @@ fn request_output_with_logprobs_and_kv(
new_token_ids,
new_logprobs: new_logprobs.map(MaybeWireLogprobs::Direct),
new_prompt_logprobs_tensors: prompt_logprobs.map(MaybeWireLogprobs::Direct),
pooling_output: None,
finish_reason,
stop_reason: None,
events: None,
kv_transfer_params,
ec_transfer_params,
trace_headers: None,
prefill_stats: None,
routed_experts: None,
num_nans_in_logits: 0,
..Default::default()
}
}
+1 -11
View File
@@ -114,18 +114,8 @@ fn request_output(
EngineCoreOutput {
request_id: request_id.to_string(),
new_token_ids,
new_logprobs: None,
new_prompt_logprobs_tensors: None,
pooling_output: None,
finish_reason,
stop_reason: None,
events: None,
kv_transfer_params: None,
ec_transfer_params: None,
trace_headers: None,
prefill_stats: None,
routed_experts: None,
num_nans_in_logits: 0,
..Default::default()
}
}
@@ -96,18 +96,8 @@ fn request_output(
EngineCoreOutput {
request_id: request_id.to_string(),
new_token_ids,
new_logprobs: None,
new_prompt_logprobs_tensors: None,
pooling_output: None,
finish_reason,
stop_reason: None,
events: None,
kv_transfer_params: None,
ec_transfer_params: None,
trace_headers: None,
prefill_stats: None,
routed_experts: None,
num_nans_in_logits: 0,
..Default::default()
}
}
+3 -24
View File
@@ -76,18 +76,9 @@ fn request_output_with_stop_reason(
EngineCoreOutput {
request_id: request_id.to_string(),
new_token_ids,
new_logprobs: None,
new_prompt_logprobs_tensors: None,
pooling_output: None,
finish_reason,
stop_reason,
events: None,
kv_transfer_params: None,
ec_transfer_params: None,
trace_headers: None,
prefill_stats: None,
routed_experts: None,
num_nans_in_logits: 0,
..Default::default()
}
}
@@ -104,16 +95,9 @@ fn request_output_with_logprobs(
new_token_ids,
new_logprobs: new_logprobs.map(MaybeWireLogprobs::Direct),
new_prompt_logprobs_tensors: new_prompt_logprobs_tensors.map(MaybeWireLogprobs::Direct),
pooling_output: None,
finish_reason,
stop_reason,
events: None,
kv_transfer_params: None,
ec_transfer_params: None,
trace_headers: None,
prefill_stats: None,
routed_experts: None,
num_nans_in_logits: 0,
..Default::default()
}
}
@@ -132,16 +116,11 @@ fn request_output_with_logprobs_and_kv(
new_token_ids,
new_logprobs: new_logprobs.map(MaybeWireLogprobs::Direct),
new_prompt_logprobs_tensors: new_prompt_logprobs_tensors.map(MaybeWireLogprobs::Direct),
pooling_output: None,
finish_reason,
stop_reason,
events: None,
kv_transfer_params,
ec_transfer_params,
trace_headers: None,
prefill_stats: None,
routed_experts: None,
num_nans_in_logits: 0,
..Default::default()
}
}
+101
View File
@@ -7,11 +7,13 @@ import pytest
import torch
from vllm.config import ModelConfig, ParallelConfig, VllmConfig
from vllm.config.multimodal import MultiModalConfig
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.cache import (
BaseMultiModalProcessorCache,
BaseMultiModalReceiverCache,
MultiModalCache,
MultiModalCacheMissError,
MultiModalProcessorCacheInItem,
MultiModalProcessorCacheItem,
MultiModalProcessorCacheItemMetadata,
@@ -232,6 +234,105 @@ def test_ipc_enable_disable_consistency(is_cached_calls_per_iter):
)
class _StubModelConfig:
"""Minimal model-config stand-in: the cache classes only call
get_multimodal_config(), so this lets us construct them directly without a
real model, keeping this unit test free of any Hugging Face / network lookup.
"""
def __init__(self, mm_processor_cache_gb: float) -> None:
self._mm_config = MultiModalConfig(mm_processor_cache_gb=mm_processor_cache_gb)
def get_multimodal_config(self) -> MultiModalConfig:
return self._mm_config
def test_mm_cache_miss_raises_and_recovers():
"""A P0/P1 multimodal cache drift must be recoverable, not a hard crash.
Reproduces the production failure: the P0 sender cache believes an item is
cached on P1 (so it sends ``data=None``), but the P1 receiver cache does not
have it. The receiver must raise ``MultiModalCacheMissError`` (carrying the
hash) instead of asserting; ``P0.invalidate()`` must drop the stale shadow
entry; and resending the item with data must repopulate both caches.
Caches are built directly (not via the registry) so the test needs no real
model or network.
"""
model_config = _StubModelConfig(mm_processor_cache_gb=1)
p0 = MultiModalProcessorSenderCache(model_config) # type: ignore[arg-type]
p1 = MultiModalReceiverCache(model_config) # type: ignore[arg-type]
mm_hash = "image_A"
item = MultiModalKwargsItem.dummy(nbytes=64)
# Drift: the item reaches P1 with no data (P0 would have sent data=None on a
# stale HIT) but P1 does not have it. Must raise a typed, retryable error
# carrying the hash -- not assert.
with pytest.raises(MultiModalCacheMissError) as exc_info:
p1.get_and_update_item(None, mm_hash)
assert exc_info.value.mm_hashes == [mm_hash]
# Give P0 a (now stale) shadow entry for the hash.
miss = p0.get_and_update_item((item, []), mm_hash)
assert miss[0] is item # MISS path returns the data to forward to P1
assert p0.is_cached_item(mm_hash)
# On the next request P0 short-circuits to data=None -- the drift bug when
# P1 lacks the item.
hit = p0.get_and_update_item((item, []), mm_hash)
assert hit[0] is None
# Recovery: drop the stale P0 entry so the client's resend-with-data takes
# the MISS path again and repopulates P1.
p0.invalidate(mm_hash)
assert not p0.is_cached_item(mm_hash)
resent = p0.get_and_update_item((item, []), mm_hash)
assert resent[0] is item # MISS again -> data is resent
assert p1.get_and_update_item(item, mm_hash) == item # P1 now caches it
# A subsequent uuid-only request now succeeds on P1.
assert p1.get_and_update_item(None, mm_hash) == item
def test_mm_cache_miss_batches_all_drifted_hashes():
"""All hashes drifted within one request must surface in a single error.
Otherwise a request with k drifted items needs k client retries (each resend
un-shadows only the one reported hash). get_and_update_features collects every
miss and raises once, so a single retry recovers the whole request -- while the
non-drifted items in the same request are still ingested.
"""
model_config = _StubModelConfig(mm_processor_cache_gb=1)
p1 = MultiModalReceiverCache(model_config) # type: ignore[arg-type]
item = MultiModalKwargsItem.dummy(nbytes=64)
def _feature(
mm_hash: str, data: MultiModalKwargsItem | None
) -> MultiModalFeatureSpec:
return MultiModalFeatureSpec(
data=data,
modality="image",
identifier=mm_hash,
mm_position=PlaceholderRange(offset=0, length=1),
mm_hash=mm_hash,
)
# A and C carry data; B, D, E arrive data=None (stale P0 shadow HITs) but P1
# has none of them -- all three must be reported together, in order.
features = [
_feature("A", item),
_feature("B", None),
_feature("C", item),
_feature("D", None),
_feature("E", None),
]
with pytest.raises(MultiModalCacheMissError) as exc_info:
p1.get_and_update_features(features)
assert exc_info.value.mm_hashes == ["B", "D", "E"]
# The non-drifted items were still ingested into P1 before the raise.
assert "A" in p1._cache and "C" in p1._cache
def _run_test_cache_eviction_lru(
p0_cache: BaseMultiModalProcessorCache,
p1_cache: BaseMultiModalReceiverCache,
+52 -2
View File
@@ -39,6 +39,29 @@ if TYPE_CHECKING:
logger = init_logger(__name__)
class MultiModalCacheMissError(RuntimeError):
"""Raised by the P1 receiver cache when items are requested with no data and
are not cached.
P0 (frontend) keeps a metadata-only shadow of P1 (engine) and sends
``data=None`` on a shadow hit. The two caches are updated in different orders
across processes, so they can drift -- leaving P0 referencing items P1 has
evicted. Raising (instead of asserting) lets the engine return a retryable
response and have P0 drop the stale entries
(``BaseMultiModalProcessorCache.invalidate``) so the client resends the data.
Carries every drifted ``mm_hash`` in the request so P0 can drop them all in one
pass -- one retry then recovers the whole request, not one item per retry.
"""
def __init__(self, mm_hashes: list[str]) -> None:
super().__init__(
f"Multi-modal items {mm_hashes} are not in the receiver (P1) cache and "
"no data was provided to recompute them (P0/P1 cache drift); the request "
"should be retried with the multi-modal data attached."
)
self.mm_hashes = mm_hashes
class MultiModalProcessorCacheItem:
"""
The data to store inside `MultiModalProcessorOnlyCache`.
@@ -295,6 +318,13 @@ class BaseMultiModalProcessorCache(
"""
return [self.is_cached_item(mm_hash) for mm_hash in mm_hashes]
def invalidate(self, mm_hash: str) -> None:
"""Drop ``mm_hash`` from this P0 shadow cache to recover from P0/P1 drift.
No-op by default; shadow caches that can drift from P1 override this.
"""
pass
def close(self) -> None:
"""Close the underlying cache, if needed."""
pass
@@ -433,6 +463,12 @@ class MultiModalProcessorSenderCache(BaseMultiModalProcessorCache):
def make_stats(self, *, delta: bool = False) -> CacheInfo:
return self._cache.stat(delta=delta)
@override
def invalidate(self, mm_hash: str) -> None:
# Drop our stale shadow entry so the next request for this hash re-sends the
# data and repopulates P1 (see MultiModalCacheMissError).
self._cache.pop(mm_hash, None)
class ShmObjectStoreSenderCache(BaseMultiModalProcessorCache):
"""
@@ -602,9 +638,19 @@ class BaseMultiModalReceiverCache(
cache_key = feature.mm_hash or feature.identifier
self.touch_receiver_cache_item(cache_key, feature.data)
missing_mm_hashes: list[str] = []
for feature in mm_features:
cache_key = feature.mm_hash or feature.identifier
feature.data = self.get_and_update_item(feature.data, cache_key)
try:
feature.data = self.get_and_update_item(feature.data, cache_key)
except MultiModalCacheMissError as e:
# Collect every drifted hash in this request before raising, so the
# engine can have P0 invalidate them all at once -- otherwise a
# request with k drifted items needs k client retries (each resend
# only un-shadows the one reported hash).
missing_mm_hashes.extend(e.mm_hashes)
if missing_mm_hashes:
raise MultiModalCacheMissError(missing_mm_hashes)
return mm_features
@abstractmethod
@@ -657,7 +703,11 @@ class MultiModalReceiverCache(BaseMultiModalReceiverCache):
if (cached_item := self._cache.get(mm_hash)) is not None:
return cached_item
assert mm_item is not None, f"Expected a cached item for {mm_hash=}"
# No data and not cached here: P0 sent data=None trusting its shadow, but
# the P0/P1 caches have drifted. Raise a retryable error (not assert) so the
# engine can have P0 drop the stale entry and the client resend the data.
if mm_item is None:
raise MultiModalCacheMissError([mm_hash])
self._cache[mm_hash] = mm_item
return mm_item
+5
View File
@@ -211,6 +211,11 @@ class EngineCoreOutput(
# The number of NaNs in logits.
# A value greater than 0 indicates that the output is corrupted.
num_nans_in_logits: int = 0
# Multi-modal hashes missing from the P1 receiver cache (P0/P1 drift; see
# `MultiModalCacheMissError`). Non-empty => retryable: the frontend drops these
# from its sender cache and the request is resent with the data. Appended last
# so `array_like` positional serialization stays backward compatible.
mm_cache_miss_hashes: list[str] | None = None
@property
def finished(self) -> bool:
+12
View File
@@ -679,6 +679,8 @@ class AsyncLLM(EngineClient):
self._logger_ref = [self.logger_manager]
logger_ref = self._logger_ref
renderer = self.renderer
# P0 multi-modal sender ("shadow") cache; None for text-only models.
mm_processor_cache = renderer.mm_processor_cache
chunk_size = envs.VLLM_V1_OUTPUT_PROC_CHUNK_SIZE
async def output_handler():
@@ -706,6 +708,16 @@ class AsyncLLM(EngineClient):
# NOTE: RequestOutputs are pushed to their queues.
assert not processed_outputs.request_outputs
# 2b) Recover from P0/P1 cache drift: the engine flags hashes
# it couldn't find (mm_cache_miss_hashes); drop them from the
# P0 shadow so the client's retry resends the data and
# repopulates P1. Hot-path no-op (field is None otherwise).
if mm_processor_cache is not None:
for eco in outputs_slice:
if eco.mm_cache_miss_hashes:
for mm_hash in eco.mm_cache_miss_hashes:
mm_processor_cache.invalidate(mm_hash)
# Allow other asyncio tasks to run between chunks
if end < num_outputs:
await asyncio.sleep(0)
+40
View File
@@ -32,6 +32,7 @@ from vllm.logger import init_logger
from vllm.logging_utils.dump_input import dump_engine_exception
from vllm.lora.request import LoRARequest
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.cache import MultiModalCacheMissError
from vllm.tasks import POOLING_TASKS, SupportedTask
from vllm.tracing import instrument, maybe_init_worker_tracer
from vllm.transformers_utils.config import maybe_register_config_serialize_by_value
@@ -1710,6 +1711,11 @@ class EngineCoreProc(EngineCore):
req: EngineCoreRequest = add_request_decoder.decode(data_frames)
try:
request = self.preprocess_add_request(req)
except MultiModalCacheMissError as e:
# P0/P1 shadow drift -- return a retryable signal (P0
# drops the stale entry, client resends with data).
self._handle_mm_cache_miss(req, e)
continue
except Exception:
self._handle_request_preproc_error(req)
continue
@@ -1803,6 +1809,40 @@ class EngineCoreProc(EngineCore):
# Limit the number of buffers to reuse.
reuse_buffers.append(buffer)
def _handle_mm_cache_miss(
self, request: EngineCoreRequest, err: MultiModalCacheMissError
) -> None:
"""Return a retryable response for a P0/P1 cache-drift miss.
Surfaces every drifted hash via ``EngineCoreOutput.mm_cache_miss_hashes`` so
the frontend drops them from its sender cache and the client resends with
data (see ``MultiModalCacheMissError``). Logged at warning, not exception,
because it is expected and self-healing.
"""
logger.warning(
"Multi-modal cache miss for request %s (mm_hashes=%s): P0/P1 cache "
"drift; returning a retryable response so the items are resent with data.",
request.request_id,
err.mm_hashes,
)
self.output_queue.put_nowait(
(
request.client_index,
EngineCoreOutputs(
engine_index=self.engine_index,
finished_requests={request.request_id},
outputs=[
EngineCoreOutput(
request_id=request.request_id,
new_token_ids=[],
finish_reason=FinishReason.ERROR,
mm_cache_miss_hashes=err.mm_hashes,
)
],
),
)
)
@staticmethod
def _send_msg_tracking_payload(
socket: zmq.Socket, buffers: Sequence[bytestr]