Signed-off-by: William Zhao <williamzh@meta.com> Signed-off-by: WillZZZy <93454470+WillZZZy@users.noreply.github.com> Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk> Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
co-authored by
Cyrus Leung
Claude
parent
f4e9bb01de
commit
3962042304
@@ -46,18 +46,9 @@ fn request_output(
|
||||
EngineCoreOutput {
|
||||
request_id: request_id.to_string(),
|
||||
new_token_ids,
|
||||
new_logprobs: None,
|
||||
new_prompt_logprobs_tensors: None,
|
||||
pooling_output: None,
|
||||
finish_reason,
|
||||
stop_reason,
|
||||
events: None,
|
||||
kv_transfer_params: None,
|
||||
ec_transfer_params: None,
|
||||
trace_headers: None,
|
||||
prefill_stats: None,
|
||||
routed_experts: None,
|
||||
num_nans_in_logits: 0,
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -74,16 +65,9 @@ fn request_output_with_logprobs(
|
||||
new_token_ids,
|
||||
new_logprobs: new_logprobs.map(MaybeWireLogprobs::Direct),
|
||||
new_prompt_logprobs_tensors: new_prompt_logprobs_tensors.map(MaybeWireLogprobs::Direct),
|
||||
pooling_output: None,
|
||||
finish_reason,
|
||||
stop_reason,
|
||||
events: None,
|
||||
kv_transfer_params: None,
|
||||
ec_transfer_params: None,
|
||||
trace_headers: None,
|
||||
prefill_stats: None,
|
||||
routed_experts: None,
|
||||
num_nans_in_logits: 0,
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -112,6 +112,17 @@ pub struct EngineCoreOutput {
|
||||
/// Number of NaNs seen in logits. Values above zero indicate corruption.
|
||||
#[serde(default)]
|
||||
pub num_nans_in_logits: u32,
|
||||
/// Multi-modal hashes the engine could not find in its receiver cache,
|
||||
/// because the frontend sent `data: None` for an item the engine had
|
||||
/// already evicted. Non-empty makes the output retryable: a frontend that
|
||||
/// keeps a metadata-only shadow of the engine cache drops these entries and
|
||||
/// resends the request with the item data attached.
|
||||
///
|
||||
/// TODO: always `None` here, since this client has no multi-modal processor
|
||||
/// cache and always sends item data inline. Act on it once the Rust
|
||||
/// frontend grows one.
|
||||
#[serde(default)]
|
||||
pub mm_cache_miss_hashes: Option<Vec<String>>,
|
||||
}
|
||||
|
||||
impl EngineCoreOutput {
|
||||
@@ -372,18 +383,9 @@ mod tests {
|
||||
outputs: vec![EngineCoreOutput {
|
||||
request_id: "req-1".to_string(),
|
||||
new_token_ids: vec![42],
|
||||
new_logprobs: None,
|
||||
new_prompt_logprobs_tensors: None,
|
||||
pooling_output: None,
|
||||
finish_reason: Some(EngineCoreFinishReason::Length),
|
||||
stop_reason: Some(StopReason::Text("stop".to_string())),
|
||||
events: None,
|
||||
kv_transfer_params: None,
|
||||
ec_transfer_params: None,
|
||||
trace_headers: None,
|
||||
prefill_stats: None,
|
||||
routed_experts: None,
|
||||
num_nans_in_logits: 0,
|
||||
..Default::default()
|
||||
}],
|
||||
finished_requests: Some(BTreeSet::from(["req-1".to_string()])),
|
||||
..Default::default()
|
||||
@@ -437,6 +439,7 @@ mod tests {
|
||||
prefill_stats: None,
|
||||
routed_experts: None,
|
||||
num_nans_in_logits: 0,
|
||||
mm_cache_miss_hashes: None,
|
||||
},
|
||||
],
|
||||
scheduler_stats: None,
|
||||
|
||||
@@ -223,18 +223,8 @@ fn request_output(
|
||||
EngineCoreOutput {
|
||||
request_id: request_id.to_string(),
|
||||
new_token_ids,
|
||||
new_logprobs: None,
|
||||
new_prompt_logprobs_tensors: None,
|
||||
pooling_output: None,
|
||||
finish_reason,
|
||||
stop_reason: None,
|
||||
events: None,
|
||||
kv_transfer_params: None,
|
||||
ec_transfer_params: None,
|
||||
trace_headers: None,
|
||||
prefill_stats: None,
|
||||
routed_experts: None,
|
||||
num_nans_in_logits: 0,
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2607,6 +2597,7 @@ fn python_msgpack_fixtures_match_rust_encoding() {
|
||||
prefill_stats: None,
|
||||
routed_experts: None,
|
||||
num_nans_in_logits: 0,
|
||||
mm_cache_miss_hashes: None,
|
||||
},
|
||||
],
|
||||
scheduler_stats: None,
|
||||
|
||||
@@ -97,6 +97,7 @@ class EngineCoreOutput(
|
||||
prefill_stats: object | None = None
|
||||
routed_experts: object | None = None
|
||||
num_nans_in_logits: int = 0
|
||||
mm_cache_miss_hashes: list[str] | None = None
|
||||
|
||||
|
||||
class EngineCoreOutputs(
|
||||
|
||||
@@ -47,18 +47,9 @@ fn request_output_with_events(
|
||||
EngineCoreOutput {
|
||||
request_id: request_id.to_string(),
|
||||
new_token_ids,
|
||||
new_logprobs: None,
|
||||
new_prompt_logprobs_tensors: None,
|
||||
pooling_output: None,
|
||||
finish_reason,
|
||||
stop_reason: None,
|
||||
events,
|
||||
kv_transfer_params: None,
|
||||
ec_transfer_params: None,
|
||||
trace_headers: None,
|
||||
prefill_stats: None,
|
||||
routed_experts: None,
|
||||
num_nans_in_logits: 0,
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -74,16 +65,8 @@ fn request_output_with_logprobs(
|
||||
new_token_ids,
|
||||
new_logprobs: new_logprobs.map(MaybeWireLogprobs::Direct),
|
||||
new_prompt_logprobs_tensors: prompt_logprobs.map(MaybeWireLogprobs::Direct),
|
||||
pooling_output: None,
|
||||
finish_reason,
|
||||
stop_reason: None,
|
||||
events: None,
|
||||
kv_transfer_params: None,
|
||||
ec_transfer_params: None,
|
||||
trace_headers: None,
|
||||
prefill_stats: None,
|
||||
routed_experts: None,
|
||||
num_nans_in_logits: 0,
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -101,16 +84,10 @@ fn request_output_with_logprobs_and_kv(
|
||||
new_token_ids,
|
||||
new_logprobs: new_logprobs.map(MaybeWireLogprobs::Direct),
|
||||
new_prompt_logprobs_tensors: prompt_logprobs.map(MaybeWireLogprobs::Direct),
|
||||
pooling_output: None,
|
||||
finish_reason,
|
||||
stop_reason: None,
|
||||
events: None,
|
||||
kv_transfer_params,
|
||||
ec_transfer_params,
|
||||
trace_headers: None,
|
||||
prefill_stats: None,
|
||||
routed_experts: None,
|
||||
num_nans_in_logits: 0,
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -114,18 +114,8 @@ fn request_output(
|
||||
EngineCoreOutput {
|
||||
request_id: request_id.to_string(),
|
||||
new_token_ids,
|
||||
new_logprobs: None,
|
||||
new_prompt_logprobs_tensors: None,
|
||||
pooling_output: None,
|
||||
finish_reason,
|
||||
stop_reason: None,
|
||||
events: None,
|
||||
kv_transfer_params: None,
|
||||
ec_transfer_params: None,
|
||||
trace_headers: None,
|
||||
prefill_stats: None,
|
||||
routed_experts: None,
|
||||
num_nans_in_logits: 0,
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -96,18 +96,8 @@ fn request_output(
|
||||
EngineCoreOutput {
|
||||
request_id: request_id.to_string(),
|
||||
new_token_ids,
|
||||
new_logprobs: None,
|
||||
new_prompt_logprobs_tensors: None,
|
||||
pooling_output: None,
|
||||
finish_reason,
|
||||
stop_reason: None,
|
||||
events: None,
|
||||
kv_transfer_params: None,
|
||||
ec_transfer_params: None,
|
||||
trace_headers: None,
|
||||
prefill_stats: None,
|
||||
routed_experts: None,
|
||||
num_nans_in_logits: 0,
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -76,18 +76,9 @@ fn request_output_with_stop_reason(
|
||||
EngineCoreOutput {
|
||||
request_id: request_id.to_string(),
|
||||
new_token_ids,
|
||||
new_logprobs: None,
|
||||
new_prompt_logprobs_tensors: None,
|
||||
pooling_output: None,
|
||||
finish_reason,
|
||||
stop_reason,
|
||||
events: None,
|
||||
kv_transfer_params: None,
|
||||
ec_transfer_params: None,
|
||||
trace_headers: None,
|
||||
prefill_stats: None,
|
||||
routed_experts: None,
|
||||
num_nans_in_logits: 0,
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -104,16 +95,9 @@ fn request_output_with_logprobs(
|
||||
new_token_ids,
|
||||
new_logprobs: new_logprobs.map(MaybeWireLogprobs::Direct),
|
||||
new_prompt_logprobs_tensors: new_prompt_logprobs_tensors.map(MaybeWireLogprobs::Direct),
|
||||
pooling_output: None,
|
||||
finish_reason,
|
||||
stop_reason,
|
||||
events: None,
|
||||
kv_transfer_params: None,
|
||||
ec_transfer_params: None,
|
||||
trace_headers: None,
|
||||
prefill_stats: None,
|
||||
routed_experts: None,
|
||||
num_nans_in_logits: 0,
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -132,16 +116,11 @@ fn request_output_with_logprobs_and_kv(
|
||||
new_token_ids,
|
||||
new_logprobs: new_logprobs.map(MaybeWireLogprobs::Direct),
|
||||
new_prompt_logprobs_tensors: new_prompt_logprobs_tensors.map(MaybeWireLogprobs::Direct),
|
||||
pooling_output: None,
|
||||
finish_reason,
|
||||
stop_reason,
|
||||
events: None,
|
||||
kv_transfer_params,
|
||||
ec_transfer_params,
|
||||
trace_headers: None,
|
||||
prefill_stats: None,
|
||||
routed_experts: None,
|
||||
num_nans_in_logits: 0,
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -7,11 +7,13 @@ import pytest
|
||||
import torch
|
||||
|
||||
from vllm.config import ModelConfig, ParallelConfig, VllmConfig
|
||||
from vllm.config.multimodal import MultiModalConfig
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.cache import (
|
||||
BaseMultiModalProcessorCache,
|
||||
BaseMultiModalReceiverCache,
|
||||
MultiModalCache,
|
||||
MultiModalCacheMissError,
|
||||
MultiModalProcessorCacheInItem,
|
||||
MultiModalProcessorCacheItem,
|
||||
MultiModalProcessorCacheItemMetadata,
|
||||
@@ -232,6 +234,105 @@ def test_ipc_enable_disable_consistency(is_cached_calls_per_iter):
|
||||
)
|
||||
|
||||
|
||||
class _StubModelConfig:
|
||||
"""Minimal model-config stand-in: the cache classes only call
|
||||
get_multimodal_config(), so this lets us construct them directly without a
|
||||
real model, keeping this unit test free of any Hugging Face / network lookup.
|
||||
"""
|
||||
|
||||
def __init__(self, mm_processor_cache_gb: float) -> None:
|
||||
self._mm_config = MultiModalConfig(mm_processor_cache_gb=mm_processor_cache_gb)
|
||||
|
||||
def get_multimodal_config(self) -> MultiModalConfig:
|
||||
return self._mm_config
|
||||
|
||||
|
||||
def test_mm_cache_miss_raises_and_recovers():
|
||||
"""A P0/P1 multimodal cache drift must be recoverable, not a hard crash.
|
||||
|
||||
Reproduces the production failure: the P0 sender cache believes an item is
|
||||
cached on P1 (so it sends ``data=None``), but the P1 receiver cache does not
|
||||
have it. The receiver must raise ``MultiModalCacheMissError`` (carrying the
|
||||
hash) instead of asserting; ``P0.invalidate()`` must drop the stale shadow
|
||||
entry; and resending the item with data must repopulate both caches.
|
||||
|
||||
Caches are built directly (not via the registry) so the test needs no real
|
||||
model or network.
|
||||
"""
|
||||
model_config = _StubModelConfig(mm_processor_cache_gb=1)
|
||||
p0 = MultiModalProcessorSenderCache(model_config) # type: ignore[arg-type]
|
||||
p1 = MultiModalReceiverCache(model_config) # type: ignore[arg-type]
|
||||
|
||||
mm_hash = "image_A"
|
||||
item = MultiModalKwargsItem.dummy(nbytes=64)
|
||||
|
||||
# Drift: the item reaches P1 with no data (P0 would have sent data=None on a
|
||||
# stale HIT) but P1 does not have it. Must raise a typed, retryable error
|
||||
# carrying the hash -- not assert.
|
||||
with pytest.raises(MultiModalCacheMissError) as exc_info:
|
||||
p1.get_and_update_item(None, mm_hash)
|
||||
assert exc_info.value.mm_hashes == [mm_hash]
|
||||
|
||||
# Give P0 a (now stale) shadow entry for the hash.
|
||||
miss = p0.get_and_update_item((item, []), mm_hash)
|
||||
assert miss[0] is item # MISS path returns the data to forward to P1
|
||||
assert p0.is_cached_item(mm_hash)
|
||||
# On the next request P0 short-circuits to data=None -- the drift bug when
|
||||
# P1 lacks the item.
|
||||
hit = p0.get_and_update_item((item, []), mm_hash)
|
||||
assert hit[0] is None
|
||||
|
||||
# Recovery: drop the stale P0 entry so the client's resend-with-data takes
|
||||
# the MISS path again and repopulates P1.
|
||||
p0.invalidate(mm_hash)
|
||||
assert not p0.is_cached_item(mm_hash)
|
||||
|
||||
resent = p0.get_and_update_item((item, []), mm_hash)
|
||||
assert resent[0] is item # MISS again -> data is resent
|
||||
assert p1.get_and_update_item(item, mm_hash) == item # P1 now caches it
|
||||
# A subsequent uuid-only request now succeeds on P1.
|
||||
assert p1.get_and_update_item(None, mm_hash) == item
|
||||
|
||||
|
||||
def test_mm_cache_miss_batches_all_drifted_hashes():
|
||||
"""All hashes drifted within one request must surface in a single error.
|
||||
|
||||
Otherwise a request with k drifted items needs k client retries (each resend
|
||||
un-shadows only the one reported hash). get_and_update_features collects every
|
||||
miss and raises once, so a single retry recovers the whole request -- while the
|
||||
non-drifted items in the same request are still ingested.
|
||||
"""
|
||||
model_config = _StubModelConfig(mm_processor_cache_gb=1)
|
||||
p1 = MultiModalReceiverCache(model_config) # type: ignore[arg-type]
|
||||
item = MultiModalKwargsItem.dummy(nbytes=64)
|
||||
|
||||
def _feature(
|
||||
mm_hash: str, data: MultiModalKwargsItem | None
|
||||
) -> MultiModalFeatureSpec:
|
||||
return MultiModalFeatureSpec(
|
||||
data=data,
|
||||
modality="image",
|
||||
identifier=mm_hash,
|
||||
mm_position=PlaceholderRange(offset=0, length=1),
|
||||
mm_hash=mm_hash,
|
||||
)
|
||||
|
||||
# A and C carry data; B, D, E arrive data=None (stale P0 shadow HITs) but P1
|
||||
# has none of them -- all three must be reported together, in order.
|
||||
features = [
|
||||
_feature("A", item),
|
||||
_feature("B", None),
|
||||
_feature("C", item),
|
||||
_feature("D", None),
|
||||
_feature("E", None),
|
||||
]
|
||||
with pytest.raises(MultiModalCacheMissError) as exc_info:
|
||||
p1.get_and_update_features(features)
|
||||
assert exc_info.value.mm_hashes == ["B", "D", "E"]
|
||||
# The non-drifted items were still ingested into P1 before the raise.
|
||||
assert "A" in p1._cache and "C" in p1._cache
|
||||
|
||||
|
||||
def _run_test_cache_eviction_lru(
|
||||
p0_cache: BaseMultiModalProcessorCache,
|
||||
p1_cache: BaseMultiModalReceiverCache,
|
||||
|
||||
@@ -39,6 +39,29 @@ if TYPE_CHECKING:
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class MultiModalCacheMissError(RuntimeError):
|
||||
"""Raised by the P1 receiver cache when items are requested with no data and
|
||||
are not cached.
|
||||
|
||||
P0 (frontend) keeps a metadata-only shadow of P1 (engine) and sends
|
||||
``data=None`` on a shadow hit. The two caches are updated in different orders
|
||||
across processes, so they can drift -- leaving P0 referencing items P1 has
|
||||
evicted. Raising (instead of asserting) lets the engine return a retryable
|
||||
response and have P0 drop the stale entries
|
||||
(``BaseMultiModalProcessorCache.invalidate``) so the client resends the data.
|
||||
Carries every drifted ``mm_hash`` in the request so P0 can drop them all in one
|
||||
pass -- one retry then recovers the whole request, not one item per retry.
|
||||
"""
|
||||
|
||||
def __init__(self, mm_hashes: list[str]) -> None:
|
||||
super().__init__(
|
||||
f"Multi-modal items {mm_hashes} are not in the receiver (P1) cache and "
|
||||
"no data was provided to recompute them (P0/P1 cache drift); the request "
|
||||
"should be retried with the multi-modal data attached."
|
||||
)
|
||||
self.mm_hashes = mm_hashes
|
||||
|
||||
|
||||
class MultiModalProcessorCacheItem:
|
||||
"""
|
||||
The data to store inside `MultiModalProcessorOnlyCache`.
|
||||
@@ -295,6 +318,13 @@ class BaseMultiModalProcessorCache(
|
||||
"""
|
||||
return [self.is_cached_item(mm_hash) for mm_hash in mm_hashes]
|
||||
|
||||
def invalidate(self, mm_hash: str) -> None:
|
||||
"""Drop ``mm_hash`` from this P0 shadow cache to recover from P0/P1 drift.
|
||||
|
||||
No-op by default; shadow caches that can drift from P1 override this.
|
||||
"""
|
||||
pass
|
||||
|
||||
def close(self) -> None:
|
||||
"""Close the underlying cache, if needed."""
|
||||
pass
|
||||
@@ -433,6 +463,12 @@ class MultiModalProcessorSenderCache(BaseMultiModalProcessorCache):
|
||||
def make_stats(self, *, delta: bool = False) -> CacheInfo:
|
||||
return self._cache.stat(delta=delta)
|
||||
|
||||
@override
|
||||
def invalidate(self, mm_hash: str) -> None:
|
||||
# Drop our stale shadow entry so the next request for this hash re-sends the
|
||||
# data and repopulates P1 (see MultiModalCacheMissError).
|
||||
self._cache.pop(mm_hash, None)
|
||||
|
||||
|
||||
class ShmObjectStoreSenderCache(BaseMultiModalProcessorCache):
|
||||
"""
|
||||
@@ -602,9 +638,19 @@ class BaseMultiModalReceiverCache(
|
||||
cache_key = feature.mm_hash or feature.identifier
|
||||
self.touch_receiver_cache_item(cache_key, feature.data)
|
||||
|
||||
missing_mm_hashes: list[str] = []
|
||||
for feature in mm_features:
|
||||
cache_key = feature.mm_hash or feature.identifier
|
||||
feature.data = self.get_and_update_item(feature.data, cache_key)
|
||||
try:
|
||||
feature.data = self.get_and_update_item(feature.data, cache_key)
|
||||
except MultiModalCacheMissError as e:
|
||||
# Collect every drifted hash in this request before raising, so the
|
||||
# engine can have P0 invalidate them all at once -- otherwise a
|
||||
# request with k drifted items needs k client retries (each resend
|
||||
# only un-shadows the one reported hash).
|
||||
missing_mm_hashes.extend(e.mm_hashes)
|
||||
if missing_mm_hashes:
|
||||
raise MultiModalCacheMissError(missing_mm_hashes)
|
||||
return mm_features
|
||||
|
||||
@abstractmethod
|
||||
@@ -657,7 +703,11 @@ class MultiModalReceiverCache(BaseMultiModalReceiverCache):
|
||||
if (cached_item := self._cache.get(mm_hash)) is not None:
|
||||
return cached_item
|
||||
|
||||
assert mm_item is not None, f"Expected a cached item for {mm_hash=}"
|
||||
# No data and not cached here: P0 sent data=None trusting its shadow, but
|
||||
# the P0/P1 caches have drifted. Raise a retryable error (not assert) so the
|
||||
# engine can have P0 drop the stale entry and the client resend the data.
|
||||
if mm_item is None:
|
||||
raise MultiModalCacheMissError([mm_hash])
|
||||
|
||||
self._cache[mm_hash] = mm_item
|
||||
return mm_item
|
||||
|
||||
@@ -211,6 +211,11 @@ class EngineCoreOutput(
|
||||
# The number of NaNs in logits.
|
||||
# A value greater than 0 indicates that the output is corrupted.
|
||||
num_nans_in_logits: int = 0
|
||||
# Multi-modal hashes missing from the P1 receiver cache (P0/P1 drift; see
|
||||
# `MultiModalCacheMissError`). Non-empty => retryable: the frontend drops these
|
||||
# from its sender cache and the request is resent with the data. Appended last
|
||||
# so `array_like` positional serialization stays backward compatible.
|
||||
mm_cache_miss_hashes: list[str] | None = None
|
||||
|
||||
@property
|
||||
def finished(self) -> bool:
|
||||
|
||||
@@ -679,6 +679,8 @@ class AsyncLLM(EngineClient):
|
||||
self._logger_ref = [self.logger_manager]
|
||||
logger_ref = self._logger_ref
|
||||
renderer = self.renderer
|
||||
# P0 multi-modal sender ("shadow") cache; None for text-only models.
|
||||
mm_processor_cache = renderer.mm_processor_cache
|
||||
chunk_size = envs.VLLM_V1_OUTPUT_PROC_CHUNK_SIZE
|
||||
|
||||
async def output_handler():
|
||||
@@ -706,6 +708,16 @@ class AsyncLLM(EngineClient):
|
||||
# NOTE: RequestOutputs are pushed to their queues.
|
||||
assert not processed_outputs.request_outputs
|
||||
|
||||
# 2b) Recover from P0/P1 cache drift: the engine flags hashes
|
||||
# it couldn't find (mm_cache_miss_hashes); drop them from the
|
||||
# P0 shadow so the client's retry resends the data and
|
||||
# repopulates P1. Hot-path no-op (field is None otherwise).
|
||||
if mm_processor_cache is not None:
|
||||
for eco in outputs_slice:
|
||||
if eco.mm_cache_miss_hashes:
|
||||
for mm_hash in eco.mm_cache_miss_hashes:
|
||||
mm_processor_cache.invalidate(mm_hash)
|
||||
|
||||
# Allow other asyncio tasks to run between chunks
|
||||
if end < num_outputs:
|
||||
await asyncio.sleep(0)
|
||||
|
||||
@@ -32,6 +32,7 @@ from vllm.logger import init_logger
|
||||
from vllm.logging_utils.dump_input import dump_engine_exception
|
||||
from vllm.lora.request import LoRARequest
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.cache import MultiModalCacheMissError
|
||||
from vllm.tasks import POOLING_TASKS, SupportedTask
|
||||
from vllm.tracing import instrument, maybe_init_worker_tracer
|
||||
from vllm.transformers_utils.config import maybe_register_config_serialize_by_value
|
||||
@@ -1710,6 +1711,11 @@ class EngineCoreProc(EngineCore):
|
||||
req: EngineCoreRequest = add_request_decoder.decode(data_frames)
|
||||
try:
|
||||
request = self.preprocess_add_request(req)
|
||||
except MultiModalCacheMissError as e:
|
||||
# P0/P1 shadow drift -- return a retryable signal (P0
|
||||
# drops the stale entry, client resends with data).
|
||||
self._handle_mm_cache_miss(req, e)
|
||||
continue
|
||||
except Exception:
|
||||
self._handle_request_preproc_error(req)
|
||||
continue
|
||||
@@ -1803,6 +1809,40 @@ class EngineCoreProc(EngineCore):
|
||||
# Limit the number of buffers to reuse.
|
||||
reuse_buffers.append(buffer)
|
||||
|
||||
def _handle_mm_cache_miss(
|
||||
self, request: EngineCoreRequest, err: MultiModalCacheMissError
|
||||
) -> None:
|
||||
"""Return a retryable response for a P0/P1 cache-drift miss.
|
||||
|
||||
Surfaces every drifted hash via ``EngineCoreOutput.mm_cache_miss_hashes`` so
|
||||
the frontend drops them from its sender cache and the client resends with
|
||||
data (see ``MultiModalCacheMissError``). Logged at warning, not exception,
|
||||
because it is expected and self-healing.
|
||||
"""
|
||||
logger.warning(
|
||||
"Multi-modal cache miss for request %s (mm_hashes=%s): P0/P1 cache "
|
||||
"drift; returning a retryable response so the items are resent with data.",
|
||||
request.request_id,
|
||||
err.mm_hashes,
|
||||
)
|
||||
self.output_queue.put_nowait(
|
||||
(
|
||||
request.client_index,
|
||||
EngineCoreOutputs(
|
||||
engine_index=self.engine_index,
|
||||
finished_requests={request.request_id},
|
||||
outputs=[
|
||||
EngineCoreOutput(
|
||||
request_id=request.request_id,
|
||||
new_token_ids=[],
|
||||
finish_reason=FinishReason.ERROR,
|
||||
mm_cache_miss_hashes=err.mm_hashes,
|
||||
)
|
||||
],
|
||||
),
|
||||
)
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _send_msg_tracking_payload(
|
||||
socket: zmq.Socket, buffers: Sequence[bytestr]
|
||||
|
||||
Reference in New Issue
Block a user