[Bugfix][Multimodal] Handle prefix-covered items in SHM worker cache (#54994)
Signed-off-by: waizuichougou <2082431897@qq.com> Signed-off-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
co-authored by
Nick Hill
parent
27a94d1ce4
commit
3f1af35e01
@@ -1,6 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import multiprocessing as mp
|
||||
from types import SimpleNamespace
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
@@ -362,6 +363,76 @@ def test_mm_cache_miss_batches_all_drifted_hashes():
|
||||
assert "A" in p1._cache and "C" in p1._cache
|
||||
|
||||
|
||||
def test_shm_receiver_handles_prefix_covered_items(monkeypatch):
|
||||
"""Regression test for the EngineCore crash with prefix caching + SHM cache.
|
||||
|
||||
On a repeated request, prefix caching can fully cover an item's
|
||||
placeholder range, so strip_covered_mm_data() drops its payload: uncached
|
||||
items arrive at the worker with data=None, while SHM-cached items keep
|
||||
their address descriptor. Before the fix the SHM receiver asserted on the
|
||||
None item, terminating EngineCore on the second identical request
|
||||
(vllm-project/vllm#54994). The address item must still be resolved and
|
||||
acknowledged so the object stays reclaimable.
|
||||
"""
|
||||
monkeypatch.setenv(
|
||||
"VLLM_OBJECT_STORAGE_SHM_BUFFER_NAME", "test_shm_prefix_covered_items"
|
||||
)
|
||||
vllm_config = SimpleNamespace(
|
||||
parallel_config=SimpleNamespace(world_size=1),
|
||||
model_config=_StubModelConfig(mm_processor_cache_gb=4 * MiB_bytes / GiB_bytes),
|
||||
)
|
||||
p0 = ShmObjectStoreSenderCache(vllm_config) # type: ignore[arg-type]
|
||||
p1 = ShmObjectStoreReceiverCache(vllm_config, mp.Lock()) # type: ignore[arg-type]
|
||||
|
||||
def _feature(
|
||||
mm_hash: str, data: MultiModalKwargsItem | None
|
||||
) -> MultiModalFeatureSpec:
|
||||
return MultiModalFeatureSpec(
|
||||
data=data,
|
||||
modality="image",
|
||||
identifier=mm_hash,
|
||||
mm_position=PlaceholderRange(offset=0, length=100),
|
||||
mm_hash=mm_hash,
|
||||
)
|
||||
|
||||
mm_hash = "image_A"
|
||||
item = MultiModalKwargsItem.dummy(nbytes=1024)
|
||||
|
||||
try:
|
||||
# Request 1 (sender miss): the payload is put into SHM; the worker
|
||||
# receives an address descriptor and resolves it.
|
||||
p0.touch_sender_cache_item(mm_hash)
|
||||
address_item, _ = p0.get_and_update_item((item, []), mm_hash)
|
||||
first = _feature(mm_hash, address_item)
|
||||
p1.get_and_update_features([first])
|
||||
assert torch.equal(first.data["dummy"].data, item["dummy"].data)
|
||||
|
||||
# Request 2 (identical, fully prefix-covered): the sender hit takes
|
||||
# writer references (touch + get_cached) and returns the address
|
||||
# descriptor; the scheduler strips the covered uncached payload to
|
||||
# data=None.
|
||||
p0.touch_sender_cache_item(mm_hash)
|
||||
address_item_2, _ = p0.get_and_update_item(None, mm_hash)
|
||||
covered_uncached = _feature("image_B", None)
|
||||
covered_cached = _feature(mm_hash, address_item_2)
|
||||
|
||||
# Pre-fix this asserted in touch_receiver_cache_item and terminated
|
||||
# EngineCore.
|
||||
p1.get_and_update_features([covered_uncached, covered_cached])
|
||||
|
||||
assert covered_uncached.data is None
|
||||
# The address item is resolved to the cached payload.
|
||||
assert torch.equal(covered_cached.data["dummy"].data, item["dummy"].data)
|
||||
|
||||
# The hit's writer references were acknowledged by the worker, so the
|
||||
# object remains reclaimable instead of permanently protected.
|
||||
p0._shm_cache.free_unused()
|
||||
assert not p0._shm_cache.is_cached(mm_hash)
|
||||
finally:
|
||||
p1._shm_cache.close()
|
||||
p0.close()
|
||||
|
||||
|
||||
def _run_test_cache_eviction_lru(
|
||||
p0_cache: BaseMultiModalProcessorCache,
|
||||
p1_cache: BaseMultiModalReceiverCache,
|
||||
|
||||
@@ -139,3 +139,26 @@ def test_strip_covered_mm_data_mrope() -> None:
|
||||
assert set(stripped[1].data.keys()) == {"pixel_values", "image_grid_thw"}
|
||||
# original list is not mutated
|
||||
assert set(covered.data.keys()) == {"pixel_values", "image_grid_thw"}
|
||||
|
||||
|
||||
def test_strip_covered_mm_data_shm_address_item() -> None:
|
||||
"""SHM address descriptors must survive stripping even when covered: the
|
||||
worker needs the address to resolve the payload and to acknowledge the
|
||||
sender's writer reference. Stripping it crashed the SHM receiver cache on
|
||||
the second identical request (vllm-project/vllm#54994)."""
|
||||
address_item = MultiModalKwargsItem(
|
||||
{
|
||||
"address": MultiModalFieldElem(data=4096, field=MultiModalBatchedField()),
|
||||
"monotonic_id": MultiModalFieldElem(data=7, field=MultiModalBatchedField()),
|
||||
}
|
||||
)
|
||||
feature = MultiModalFeatureSpec(
|
||||
data=address_item,
|
||||
mm_position=PlaceholderRange(offset=0, length=100),
|
||||
identifier="shm_item",
|
||||
modality="image",
|
||||
)
|
||||
|
||||
stripped = strip_covered_mm_data([feature], num_computed_tokens=250)
|
||||
|
||||
assert stripped[0].data is address_item
|
||||
|
||||
@@ -794,6 +794,18 @@ class ShmObjectStoreReceiverCache(BaseMultiModalReceiverCache):
|
||||
reader_lock=shared_worker_lock,
|
||||
)
|
||||
|
||||
@override
|
||||
def get_and_update_features(
|
||||
self, mm_features: list["MultiModalFeatureSpec"]
|
||||
) -> list["MultiModalFeatureSpec"]:
|
||||
# strip_covered_mm_data preserves address items, so None here represents
|
||||
# a stripped uncached payload and has no SHM reference to acknowledge.
|
||||
features_with_data = [
|
||||
feature for feature in mm_features if feature.data is not None
|
||||
]
|
||||
super().get_and_update_features(features_with_data)
|
||||
return mm_features
|
||||
|
||||
@override
|
||||
def get_and_update_item(
|
||||
self,
|
||||
|
||||
@@ -221,15 +221,20 @@ def strip_covered_mm_data(
|
||||
"""Drop the tensor data of mm items whose placeholder span is fully inside
|
||||
a prefix-cache-covered region: no encoder run can be scheduled for them,
|
||||
so the workers never consume the payload fields. M-RoPE and XD-RoPE models
|
||||
are the exception: the worker computes positions for the whole prompt from
|
||||
the CPU-side metadata fields (e.g. grid dims), so those are kept. The
|
||||
keep CPU-side metadata fields used to compute positions. SHM address items
|
||||
are also kept so workers can balance the sender's reference count. The
|
||||
scheduler-side ``Request`` keeps the full features."""
|
||||
if not mm_features or num_computed_tokens == 0:
|
||||
return mm_features
|
||||
|
||||
def maybe_strip(f: MultiModalFeatureSpec) -> MultiModalFeatureSpec:
|
||||
if f.data is None or (
|
||||
f.mm_position.offset + f.mm_position.length > num_computed_tokens
|
||||
if (
|
||||
f.data is None
|
||||
# SHM address items must reach the worker so it can acknowledge
|
||||
# the sender's reference count before the item is evicted. This
|
||||
# mirrors how the SHM receiver identifies them ("address" in item).
|
||||
or "address" in f.data
|
||||
or (f.mm_position.offset + f.mm_position.length > num_computed_tokens)
|
||||
):
|
||||
return f
|
||||
|
||||
|
||||
Reference in New Issue
Block a user