[Bugfix][Multimodal] Handle prefix-covered items in SHM worker cache (#54994)

Signed-off-by: waizuichougou <2082431897@qq.com>
Signed-off-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
waizuichougou
2026-09-03 06:39:34 +00:00
committed by GitHub
co-authored by Nick Hill
parent 27a94d1ce4
commit 3f1af35e01
4 changed files with 115 additions and 4 deletions
+71
View File
@@ -1,6 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import multiprocessing as mp
from types import SimpleNamespace
import numpy as np
import pytest
@@ -362,6 +363,76 @@ def test_mm_cache_miss_batches_all_drifted_hashes():
assert "A" in p1._cache and "C" in p1._cache
def test_shm_receiver_handles_prefix_covered_items(monkeypatch):
"""Regression test for the EngineCore crash with prefix caching + SHM cache.
On a repeated request, prefix caching can fully cover an item's
placeholder range, so strip_covered_mm_data() drops its payload: uncached
items arrive at the worker with data=None, while SHM-cached items keep
their address descriptor. Before the fix the SHM receiver asserted on the
None item, terminating EngineCore on the second identical request
(vllm-project/vllm#54994). The address item must still be resolved and
acknowledged so the object stays reclaimable.
"""
monkeypatch.setenv(
"VLLM_OBJECT_STORAGE_SHM_BUFFER_NAME", "test_shm_prefix_covered_items"
)
vllm_config = SimpleNamespace(
parallel_config=SimpleNamespace(world_size=1),
model_config=_StubModelConfig(mm_processor_cache_gb=4 * MiB_bytes / GiB_bytes),
)
p0 = ShmObjectStoreSenderCache(vllm_config) # type: ignore[arg-type]
p1 = ShmObjectStoreReceiverCache(vllm_config, mp.Lock()) # type: ignore[arg-type]
def _feature(
mm_hash: str, data: MultiModalKwargsItem | None
) -> MultiModalFeatureSpec:
return MultiModalFeatureSpec(
data=data,
modality="image",
identifier=mm_hash,
mm_position=PlaceholderRange(offset=0, length=100),
mm_hash=mm_hash,
)
mm_hash = "image_A"
item = MultiModalKwargsItem.dummy(nbytes=1024)
try:
# Request 1 (sender miss): the payload is put into SHM; the worker
# receives an address descriptor and resolves it.
p0.touch_sender_cache_item(mm_hash)
address_item, _ = p0.get_and_update_item((item, []), mm_hash)
first = _feature(mm_hash, address_item)
p1.get_and_update_features([first])
assert torch.equal(first.data["dummy"].data, item["dummy"].data)
# Request 2 (identical, fully prefix-covered): the sender hit takes
# writer references (touch + get_cached) and returns the address
# descriptor; the scheduler strips the covered uncached payload to
# data=None.
p0.touch_sender_cache_item(mm_hash)
address_item_2, _ = p0.get_and_update_item(None, mm_hash)
covered_uncached = _feature("image_B", None)
covered_cached = _feature(mm_hash, address_item_2)
# Pre-fix this asserted in touch_receiver_cache_item and terminated
# EngineCore.
p1.get_and_update_features([covered_uncached, covered_cached])
assert covered_uncached.data is None
# The address item is resolved to the cached payload.
assert torch.equal(covered_cached.data["dummy"].data, item["dummy"].data)
# The hit's writer references were acknowledged by the worker, so the
# object remains reclaimable instead of permanently protected.
p0._shm_cache.free_unused()
assert not p0._shm_cache.is_cached(mm_hash)
finally:
p1._shm_cache.close()
p0.close()
def _run_test_cache_eviction_lru(
p0_cache: BaseMultiModalProcessorCache,
p1_cache: BaseMultiModalReceiverCache,
+23
View File
@@ -139,3 +139,26 @@ def test_strip_covered_mm_data_mrope() -> None:
assert set(stripped[1].data.keys()) == {"pixel_values", "image_grid_thw"}
# original list is not mutated
assert set(covered.data.keys()) == {"pixel_values", "image_grid_thw"}
def test_strip_covered_mm_data_shm_address_item() -> None:
"""SHM address descriptors must survive stripping even when covered: the
worker needs the address to resolve the payload and to acknowledge the
sender's writer reference. Stripping it crashed the SHM receiver cache on
the second identical request (vllm-project/vllm#54994)."""
address_item = MultiModalKwargsItem(
{
"address": MultiModalFieldElem(data=4096, field=MultiModalBatchedField()),
"monotonic_id": MultiModalFieldElem(data=7, field=MultiModalBatchedField()),
}
)
feature = MultiModalFeatureSpec(
data=address_item,
mm_position=PlaceholderRange(offset=0, length=100),
identifier="shm_item",
modality="image",
)
stripped = strip_covered_mm_data([feature], num_computed_tokens=250)
assert stripped[0].data is address_item
+12
View File
@@ -794,6 +794,18 @@ class ShmObjectStoreReceiverCache(BaseMultiModalReceiverCache):
reader_lock=shared_worker_lock,
)
@override
def get_and_update_features(
self, mm_features: list["MultiModalFeatureSpec"]
) -> list["MultiModalFeatureSpec"]:
# strip_covered_mm_data preserves address items, so None here represents
# a stripped uncached payload and has no SHM reference to acknowledge.
features_with_data = [
feature for feature in mm_features if feature.data is not None
]
super().get_and_update_features(features_with_data)
return mm_features
@override
def get_and_update_item(
self,
+9 -4
View File
@@ -221,15 +221,20 @@ def strip_covered_mm_data(
"""Drop the tensor data of mm items whose placeholder span is fully inside
a prefix-cache-covered region: no encoder run can be scheduled for them,
so the workers never consume the payload fields. M-RoPE and XD-RoPE models
are the exception: the worker computes positions for the whole prompt from
the CPU-side metadata fields (e.g. grid dims), so those are kept. The
keep CPU-side metadata fields used to compute positions. SHM address items
are also kept so workers can balance the sender's reference count. The
scheduler-side ``Request`` keeps the full features."""
if not mm_features or num_computed_tokens == 0:
return mm_features
def maybe_strip(f: MultiModalFeatureSpec) -> MultiModalFeatureSpec:
if f.data is None or (
f.mm_position.offset + f.mm_position.length > num_computed_tokens
if (
f.data is None
# SHM address items must reach the worker so it can acknowledge
# the sender's reference count before the item is evicted. This
# mirrors how the SHM receiver identifies them ("address" in item).
or "address" in f.data
or (f.mm_position.offset + f.mm_position.length > num_computed_tokens)
):
return f