diff --git a/tests/multimodal/test_cache.py b/tests/multimodal/test_cache.py index 7c93808f28..f77dacd341 100644 --- a/tests/multimodal/test_cache.py +++ b/tests/multimodal/test_cache.py @@ -1,6 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import multiprocessing as mp +from types import SimpleNamespace import numpy as np import pytest @@ -362,6 +363,76 @@ def test_mm_cache_miss_batches_all_drifted_hashes(): assert "A" in p1._cache and "C" in p1._cache +def test_shm_receiver_handles_prefix_covered_items(monkeypatch): + """Regression test for the EngineCore crash with prefix caching + SHM cache. + + On a repeated request, prefix caching can fully cover an item's + placeholder range, so strip_covered_mm_data() drops its payload: uncached + items arrive at the worker with data=None, while SHM-cached items keep + their address descriptor. Before the fix the SHM receiver asserted on the + None item, terminating EngineCore on the second identical request + (vllm-project/vllm#54994). The address item must still be resolved and + acknowledged so the object stays reclaimable. + """ + monkeypatch.setenv( + "VLLM_OBJECT_STORAGE_SHM_BUFFER_NAME", "test_shm_prefix_covered_items" + ) + vllm_config = SimpleNamespace( + parallel_config=SimpleNamespace(world_size=1), + model_config=_StubModelConfig(mm_processor_cache_gb=4 * MiB_bytes / GiB_bytes), + ) + p0 = ShmObjectStoreSenderCache(vllm_config) # type: ignore[arg-type] + p1 = ShmObjectStoreReceiverCache(vllm_config, mp.Lock()) # type: ignore[arg-type] + + def _feature( + mm_hash: str, data: MultiModalKwargsItem | None + ) -> MultiModalFeatureSpec: + return MultiModalFeatureSpec( + data=data, + modality="image", + identifier=mm_hash, + mm_position=PlaceholderRange(offset=0, length=100), + mm_hash=mm_hash, + ) + + mm_hash = "image_A" + item = MultiModalKwargsItem.dummy(nbytes=1024) + + try: + # Request 1 (sender miss): the payload is put into SHM; the worker + # receives an address descriptor and resolves it. + p0.touch_sender_cache_item(mm_hash) + address_item, _ = p0.get_and_update_item((item, []), mm_hash) + first = _feature(mm_hash, address_item) + p1.get_and_update_features([first]) + assert torch.equal(first.data["dummy"].data, item["dummy"].data) + + # Request 2 (identical, fully prefix-covered): the sender hit takes + # writer references (touch + get_cached) and returns the address + # descriptor; the scheduler strips the covered uncached payload to + # data=None. + p0.touch_sender_cache_item(mm_hash) + address_item_2, _ = p0.get_and_update_item(None, mm_hash) + covered_uncached = _feature("image_B", None) + covered_cached = _feature(mm_hash, address_item_2) + + # Pre-fix this asserted in touch_receiver_cache_item and terminated + # EngineCore. + p1.get_and_update_features([covered_uncached, covered_cached]) + + assert covered_uncached.data is None + # The address item is resolved to the cached payload. + assert torch.equal(covered_cached.data["dummy"].data, item["dummy"].data) + + # The hit's writer references were acknowledged by the worker, so the + # object remains reclaimable instead of permanently protected. + p0._shm_cache.free_unused() + assert not p0._shm_cache.is_cached(mm_hash) + finally: + p1._shm_cache.close() + p0.close() + + def _run_test_cache_eviction_lru( p0_cache: BaseMultiModalProcessorCache, p1_cache: BaseMultiModalReceiverCache, diff --git a/tests/v1/core/test_output.py b/tests/v1/core/test_output.py index 97f954af46..15157e1e07 100644 --- a/tests/v1/core/test_output.py +++ b/tests/v1/core/test_output.py @@ -139,3 +139,26 @@ def test_strip_covered_mm_data_mrope() -> None: assert set(stripped[1].data.keys()) == {"pixel_values", "image_grid_thw"} # original list is not mutated assert set(covered.data.keys()) == {"pixel_values", "image_grid_thw"} + + +def test_strip_covered_mm_data_shm_address_item() -> None: + """SHM address descriptors must survive stripping even when covered: the + worker needs the address to resolve the payload and to acknowledge the + sender's writer reference. Stripping it crashed the SHM receiver cache on + the second identical request (vllm-project/vllm#54994).""" + address_item = MultiModalKwargsItem( + { + "address": MultiModalFieldElem(data=4096, field=MultiModalBatchedField()), + "monotonic_id": MultiModalFieldElem(data=7, field=MultiModalBatchedField()), + } + ) + feature = MultiModalFeatureSpec( + data=address_item, + mm_position=PlaceholderRange(offset=0, length=100), + identifier="shm_item", + modality="image", + ) + + stripped = strip_covered_mm_data([feature], num_computed_tokens=250) + + assert stripped[0].data is address_item diff --git a/vllm/multimodal/cache.py b/vllm/multimodal/cache.py index 5ec160d2f8..cc48b58298 100644 --- a/vllm/multimodal/cache.py +++ b/vllm/multimodal/cache.py @@ -794,6 +794,18 @@ class ShmObjectStoreReceiverCache(BaseMultiModalReceiverCache): reader_lock=shared_worker_lock, ) + @override + def get_and_update_features( + self, mm_features: list["MultiModalFeatureSpec"] + ) -> list["MultiModalFeatureSpec"]: + # strip_covered_mm_data preserves address items, so None here represents + # a stripped uncached payload and has no SHM reference to acknowledge. + features_with_data = [ + feature for feature in mm_features if feature.data is not None + ] + super().get_and_update_features(features_with_data) + return mm_features + @override def get_and_update_item( self, diff --git a/vllm/multimodal/utils.py b/vllm/multimodal/utils.py index 683b44549e..35ba5844a4 100644 --- a/vllm/multimodal/utils.py +++ b/vllm/multimodal/utils.py @@ -221,15 +221,20 @@ def strip_covered_mm_data( """Drop the tensor data of mm items whose placeholder span is fully inside a prefix-cache-covered region: no encoder run can be scheduled for them, so the workers never consume the payload fields. M-RoPE and XD-RoPE models - are the exception: the worker computes positions for the whole prompt from - the CPU-side metadata fields (e.g. grid dims), so those are kept. The + keep CPU-side metadata fields used to compute positions. SHM address items + are also kept so workers can balance the sender's reference count. The scheduler-side ``Request`` keeps the full features.""" if not mm_features or num_computed_tokens == 0: return mm_features def maybe_strip(f: MultiModalFeatureSpec) -> MultiModalFeatureSpec: - if f.data is None or ( - f.mm_position.offset + f.mm_position.length > num_computed_tokens + if ( + f.data is None + # SHM address items must reach the worker so it can acknowledge + # the sender's reference count before the item is evicted. This + # mirrors how the SHM receiver identifies them ("address" in item). + or "address" in f.data + or (f.mm_position.offset + f.mm_position.length > num_computed_tokens) ): return f