[Security] Authenticate shared-memory multimodal cache handles (#59357)

Signed-off-by: Clinton Thomas <1033162+KernelClint@users.noreply.github.com>
Co-authored-by: Lucas Bourtoule <35483370+dhalf@users.noreply.github.com>
Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
This commit is contained in:
Clinton Thomas
2026-09-30 05:16:26 -07:00
committed by GitHub
co-authored by Lucas Bourtoule Cyrus Leung
parent 72e7874fa6
commit 3d5625cc3c
8 changed files with 574 additions and 41 deletions
+3 -3
View File
@@ -108,10 +108,10 @@ class TestSingleWriterShmRingBuffer(unittest.TestCase):
# Clear the buffer
self.ring_buffer.clear()
# Check that metadata is empty and IDs reset
# Check that metadata is empty and IDs keep increasing
self.assertEqual(len(self.ring_buffer.metadata), 0)
self.assertEqual(self.ring_buffer.monotonic_id_start, 0)
self.assertEqual(self.ring_buffer.monotonic_id_end, 0)
self.assertEqual(self.ring_buffer.monotonic_id_start, 3)
self.assertEqual(self.ring_buffer.monotonic_id_end, 3)
self.assertEqual(self.ring_buffer.data_buffer_start, 0)
self.assertEqual(self.ring_buffer.data_buffer_end, 0)
+64 -5
View File
@@ -227,10 +227,68 @@ class TestSingleWriterShmObjectStorage(unittest.TestCase):
self.assertEqual(len(self.storage.id_index), 0)
self.assertEqual(len(self.storage.ring_buffer.metadata), 0)
# Verify that new items can be added after clearing
# Verify that new items can be added after clearing. IDs keep
# increasing so handles issued before the clear cannot alias new data.
address, monotonic_id = self.storage.put("new_item", "new_value")
self.assertIn("new_item", self.storage.key_index)
self.assertEqual((address, monotonic_id), (0, 0))
self.assertEqual((address, monotonic_id), (0, 5))
def _open_reader(self) -> SingleWriterShmObjectStorage:
reader = SingleWriterShmObjectStorage.create_from_handle(self.storage.handle())
self.addCleanup(reader.close)
return reader
def test_reader_get_and_touch_each_require_valid_signature(self):
address, monotonic_id = self.storage.put("key", "value")
signature = self.storage.get_signature("key")
reader = self._open_reader()
reader.touch("key", address, monotonic_id, signature)
self.assertEqual(reader.get(address, monotonic_id, signature, "key"), "value")
flipped = [signature[0] ^ 1, *signature[1:]]
for bad in (None, flipped, [signature], signature[:-1], [256] * 32):
with self.assertRaisesRegex(ValueError, "SHM handle signature"):
reader.get(address, monotonic_id, bad, "key")
with self.assertRaisesRegex(ValueError, "SHM handle signature"):
reader.touch("key", address, monotonic_id, bad)
def test_signature_is_bound_to_key_and_address(self):
address, monotonic_id = self.storage.put("key", "value")
signature = self.storage.get_signature("key")
reader = self._open_reader()
with self.assertRaisesRegex(ValueError, "SHM handle signature"):
reader.get(address, monotonic_id, signature, "other_key")
with self.assertRaisesRegex(ValueError, "SHM handle signature"):
reader.get(address + 1, monotonic_id, signature, "key")
def test_writer_rejects_handle_no_longer_in_key_index(self):
address, monotonic_id = self.storage.put("key", "value")
signature = self.storage.get_signature("key")
self.storage.verify_signature("key", address, monotonic_id, signature)
self.storage.clear()
with self.assertRaisesRegex(ValueError, "SHM handle signature"):
self.storage.verify_signature("key", address, monotonic_id, signature)
def test_handle_issued_before_clear_reads_own_data_until_reused(self):
address, monotonic_id = self.storage.put("key", "value")
signature = self.storage.get_signature("key")
reader = self._open_reader()
self.storage.clear()
# Requests still draining after a clear keep reading their own data.
self.assertEqual(reader.get(address, monotonic_id, signature, "key"), "value")
self.storage.put("key", "new_value")
with self.assertRaisesRegex(ValueError, "modified or is invalid"):
reader.get(address, monotonic_id, signature, "key")
with self.assertRaisesRegex(ValueError, "modified or is invalid"):
reader.touch("key", address, monotonic_id, signature)
# Reader process function
@@ -242,11 +300,11 @@ def reader_process(process_id, storage_handle, items_to_read):
errors = []
for key, original_value, address, monotonic_id in items_to_read:
for key, original_value, address, monotonic_id, signature in items_to_read:
time.sleep(random.random() / 100)
try:
# Read data from shared memory
retrieved_value = reader_storage.get(address, monotonic_id)
retrieved_value = reader_storage.get(address, monotonic_id, signature, key)
# Verify data integrity
assert retrieved_value == original_value
@@ -289,7 +347,8 @@ def run_multiprocess_example():
for key, value in test_data:
print(f"Storing {key}: {value}")
address, monotonic_id = storage.put(key, value)
stored_items.append((key, value, address, monotonic_id))
signature = storage.get_signature(key)
stored_items.append((key, value, address, monotonic_id, signature))
print(f" -> Stored at address {address}, ID {monotonic_id}")
print("\n--- Retrieving Data ---")
@@ -15,14 +15,25 @@ from vllm.entrypoints.openai.models.serving import OpenAIServingModels
from vllm.entrypoints.scale_out.token_in_token_out.mm_features import (
placeholder_ranges_from_engine_input,
)
from vllm.entrypoints.scale_out.token_in_token_out.mm_serde import (
encode_mm_kwargs_item,
)
from vllm.entrypoints.scale_out.token_in_token_out.protocol import (
GenerateRequest,
GenerateResponse,
MultiModalFeatures,
PlaceholderRangeInfo,
)
from vllm.entrypoints.scale_out.token_in_token_out.serving import ServingTokens
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
from vllm.exceptions import GenerationError
from vllm.logprobs import Logprob
from vllm.multimodal.inputs import PlaceholderRange
from vllm.multimodal.inputs import (
MultiModalBatchedField,
MultiModalFieldElem,
MultiModalKwargsItem,
PlaceholderRange,
)
from vllm.outputs import CompletionOutput, RequestOutput
from vllm.renderers import renderer_from_config
from vllm.renderers.online_renderer import OnlineRenderer
@@ -285,6 +296,46 @@ async def test_non_stream_error():
await serving.serve_tokens(request)
@pytest.mark.asyncio
async def test_serve_tokens_rejects_invalid_shm_handle_before_engine_handoff():
engine = _mock_engine()
engine.generate = MagicMock()
serving = _build_serving_tokens(engine)
cache = MagicMock()
cache.validate_input_item.side_effect = ValueError(
"Invalid SHM handle signature for cache key."
)
serving.online_renderer.renderer._mm_processor_cache = cache
forged_item = MultiModalKwargsItem(
{
"address": MultiModalFieldElem(data=224, field=MultiModalBatchedField()),
"monotonic_id": MultiModalFieldElem(
data=4242,
field=MultiModalBatchedField(),
),
}
)
request = GenerateRequest(
token_ids=[1, 2, 3],
sampling_params=SamplingParams(max_tokens=1),
model=MODEL_NAME,
features=MultiModalFeatures(
mm_hashes={"image": ["image_A"]},
mm_placeholders={"image": [PlaceholderRangeInfo(offset=0, length=1)]},
kwargs_data={"image": [encode_mm_kwargs_item(forged_item)]},
),
)
response = await serving.serve_tokens(request)
assert isinstance(response, ErrorResponse)
cache.validate_input_item.assert_called_once()
validated_item, validated_hash = cache.validate_input_item.call_args.args
assert validated_hash == "image_A"
assert validated_item["address"].data == 224
engine.generate.assert_not_called()
@pytest.mark.asyncio
async def test_stream_basic():
"""Streaming returns SSE chunks with correct token_ids and ends with [DONE]."""
+208
View File
@@ -29,6 +29,7 @@ from vllm.multimodal.cache.base import (
)
from vllm.multimodal.hasher import MultiModalHasher
from vllm.multimodal.inputs import (
MultiModalBatchedField,
MultiModalFeatureSpec,
MultiModalFieldElem,
MultiModalKwargsItem,
@@ -42,6 +43,7 @@ from vllm.utils.async_utils import make_async
from vllm.utils.mem_constants import GiB_bytes, MiB_bytes
from vllm.v1.engine.async_llm import AsyncLLM
from vllm.v1.engine.llm_engine import LLMEngine
from vllm.v1.serial_utils import MsgpackDecoder, MsgpackEncoder
from ..models.utils import build_model_context
from ..utils import create_new_process_for_each_test
@@ -778,6 +780,212 @@ def test_cache_eviction_shm_cache():
_run_test_cache_eviction_shm(sender_cache, receiver_cache, base_item_size=MiB_bytes)
def test_shm_receiver_requires_sender_handle_for_current_cache_key():
vllm_config = VllmConfig(
model_config=ModelConfig(
model="llava-hf/llava-onevision-qwen2-0.5b-ov-hf",
mm_processor_cache_type="shm",
mm_shm_cache_max_object_size_mb=1,
mm_processor_cache_gb=2 * MiB_bytes / GiB_bytes,
),
)
sender_cache = ShmObjectStoreSenderCache(vllm_config)
receiver_cache = ShmObjectStoreReceiverCache(vllm_config, mp.Lock())
mm_hash = "image_A"
item = MultiModalKwargsItem.dummy(1024)
try:
sender_item, _ = sender_cache.get_and_update_item((item, []), mm_hash)
valid_feature = MultiModalFeatureSpec(
data=sender_item,
modality="image",
identifier=mm_hash,
mm_position=PlaceholderRange(offset=0, length=1),
mm_hash=mm_hash,
)
receiver_cache.get_and_update_features([valid_feature])
assert valid_feature.data == item
forged_item = MultiModalKwargsItem(
{
"address": sender_item["address"],
"monotonic_id": sender_item["monotonic_id"],
}
)
forged_feature = MultiModalFeatureSpec(
data=forged_item,
modality="image",
identifier=mm_hash,
mm_position=PlaceholderRange(offset=0, length=1),
mm_hash=mm_hash,
)
with pytest.raises(ValueError, match="SHM handle signature"):
receiver_cache.get_and_update_features([forged_feature])
wrong_key = "image_B"
wrong_key_feature = MultiModalFeatureSpec(
data=sender_item,
modality="image",
identifier=wrong_key,
mm_position=PlaceholderRange(offset=0, length=1),
mm_hash=wrong_key,
)
with pytest.raises(ValueError, match="SHM handle signature"):
receiver_cache.get_and_update_features([wrong_key_feature])
finally:
receiver_cache._shm_cache.close()
sender_cache.close()
def test_shm_sender_handle_survives_engine_msgpack_round_trip():
vllm_config = VllmConfig(
model_config=ModelConfig(
model="llava-hf/llava-onevision-qwen2-0.5b-ov-hf",
mm_processor_cache_type="shm",
mm_shm_cache_max_object_size_mb=1,
mm_processor_cache_gb=2 * MiB_bytes / GiB_bytes,
),
)
sender_cache = ShmObjectStoreSenderCache(vllm_config)
receiver_cache = ShmObjectStoreReceiverCache(vllm_config, mp.Lock())
mm_hash = "image_A"
item = MultiModalKwargsItem.dummy(1024)
try:
sender_item, _ = sender_cache.get_and_update_item((item, []), mm_hash)
encoded = MsgpackEncoder().encode(sender_item)
decoded = MsgpackDecoder(MultiModalKwargsItem).decode(encoded)
feature = MultiModalFeatureSpec(
data=decoded,
modality="image",
identifier=mm_hash,
mm_position=PlaceholderRange(offset=0, length=1),
mm_hash=mm_hash,
)
receiver_cache.get_and_update_features([feature])
assert feature.data == item
finally:
receiver_cache._shm_cache.close()
sender_cache.close()
def _small_shm_vllm_config() -> VllmConfig:
return VllmConfig(
model_config=ModelConfig(
model="llava-hf/llava-onevision-qwen2-0.5b-ov-hf",
mm_processor_cache_type="shm",
mm_shm_cache_max_object_size_mb=1,
mm_processor_cache_gb=2 * MiB_bytes / GiB_bytes,
),
)
def _with_elem(item: MultiModalKwargsItem, key: str, data) -> MultiModalKwargsItem:
elems = dict(item)
elems[key] = MultiModalFieldElem(data=data, field=MultiModalBatchedField())
return MultiModalKwargsItem(elems)
def test_shm_sender_validates_client_supplied_handles():
sender_cache = ShmObjectStoreSenderCache(_small_shm_vllm_config())
mm_hash = "image_A"
try:
handle, _ = sender_cache.get_and_update_item(
(MultiModalKwargsItem.dummy(1024), []), mm_hash
)
sender_cache.validate_input_item(handle, mm_hash)
sender_cache.validate_input_item(MultiModalKwargsItem.dummy(16), mm_hash)
signature = handle["signature"].data
forged = [
_with_elem(handle, "signature", [signature[0] ^ 1, *signature[1:]]),
_with_elem(handle, "signature", [signature]),
_with_elem(handle, "address", True),
_with_elem(handle, "address", -1),
_with_elem(handle, "monotonic_id", 2**32),
MultiModalKwargsItem(
{"address": handle["address"], "monotonic_id": handle["monotonic_id"]}
),
]
for item in forged:
with pytest.raises(ValueError, match="SHM handle signature"):
sender_cache.validate_input_item(item, mm_hash)
with pytest.raises(ValueError, match="SHM handle signature"):
sender_cache.validate_input_item(handle, "image_B")
sender_cache.clear_cache()
with pytest.raises(ValueError, match="SHM handle signature"):
sender_cache.validate_input_item(handle, mm_hash)
finally:
sender_cache.close()
def test_shm_receiver_rejects_out_of_range_handle_fields():
vllm_config = _small_shm_vllm_config()
sender_cache = ShmObjectStoreSenderCache(vllm_config)
receiver_cache = ShmObjectStoreReceiverCache(vllm_config, mp.Lock())
mm_hash = "image_A"
try:
handle, _ = sender_cache.get_and_update_item(
(MultiModalKwargsItem.dummy(1024), []), mm_hash
)
for key, value in [
("address", -1),
("address", 2**64),
("monotonic_id", -1),
("monotonic_id", 2**32),
]:
with pytest.raises(ValueError, match="SHM handle signature"):
receiver_cache.get_and_update_item(
_with_elem(handle, key, value), mm_hash
)
finally:
receiver_cache._shm_cache.close()
sender_cache.close()
def test_shm_handle_issued_before_cache_clear():
vllm_config = _small_shm_vllm_config()
sender_cache = ShmObjectStoreSenderCache(vllm_config)
receiver_cache = ShmObjectStoreReceiverCache(vllm_config, mp.Lock())
mm_hash = "image_A"
item = MultiModalKwargsItem.dummy(1024)
def feature(data: MultiModalKwargsItem) -> MultiModalFeatureSpec:
return MultiModalFeatureSpec(
data=data,
modality="image",
identifier=mm_hash,
mm_position=PlaceholderRange(offset=0, length=1),
mm_hash=mm_hash,
)
try:
stale_item, _ = sender_cache.get_and_update_item((item, []), mm_hash)
sender_cache.clear_cache()
# A request still draining after the clear resolves to its own data.
draining = feature(stale_item)
receiver_cache.get_and_update_features([draining])
assert draining.data == item
sender_cache.get_and_update_item(
(MultiModalKwargsItem.dummy(2048), []),
mm_hash,
)
with pytest.raises(ValueError, match="modified or is invalid"):
receiver_cache.get_and_update_features([feature(stale_item)])
finally:
receiver_cache._shm_cache.close()
sender_cache.close()
def test_processor_cache_shared_across_loras():
"""Test that processor cache uses mm_hash to share data across LoRAs."""
model_config = ModelConfig(
@@ -1,7 +1,10 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import hashlib
import hmac
import pickle
import secrets
from abc import ABC, abstractmethod
from collections.abc import Callable, Iterable
from contextlib import contextmanager, suppress
@@ -97,6 +100,9 @@ class SingleWriterShmRingBuffer:
- Reader synchronization handled by is_free_fn callback
- Writer handles garbage collection (free_buf) based on reader feedback
Shared Memory Layout:
`[32-byte handle-signing secret][ring buffer chunks...]`
Memory Layout per Buffer Chunk:
`[4-byte monotonic_id][4-byte chunk_size][actual_data...]`
^metadata_start ^data_start
@@ -119,6 +125,8 @@ class SingleWriterShmRingBuffer:
self.SIZE_NBYTES = 4
# 4 bytes for id, 4 bytes for buffer size
self.MD_SIZE = self.ID_NBYTES + self.SIZE_NBYTES
self.AUTH_SECRET_BYTES = 32
self.DATA_OFFSET = self.AUTH_SECRET_BYTES
self.monotonic_id_end = 0
self.monotonic_id_start = 0
self.data_buffer_start = 0
@@ -129,8 +137,11 @@ class SingleWriterShmRingBuffer:
# we are creating a buffer
self.metadata: dict[int, int] = {} # monotonic_id -> start address
self.shared_memory = shared_memory.SharedMemory(
create=True, size=self.data_buffer_size, name=name
create=True,
size=self.data_buffer_size + self.DATA_OFFSET,
name=name,
)
self._init_auth_secret()
else:
# we are opening an existing buffer
# fix to https://stackoverflow.com/q/62748654/9191338
@@ -146,7 +157,9 @@ class SingleWriterShmRingBuffer:
# so the shared memory block size may be larger or equal
# to the requested size. The size parameter is ignored
# when attaching to an existing block.
assert self.shared_memory.size >= self.data_buffer_size
assert (
self.shared_memory.size >= self.data_buffer_size + self.DATA_OFFSET
)
logger.debug(
"Shared memory created/opened with name: %s, size: %d",
@@ -164,11 +177,25 @@ class SingleWriterShmRingBuffer:
"""Clear the ring buffer."""
assert self.is_writer, "Only the writer can clear the buffer."
self.metadata.clear()
self.monotonic_id_end = 0
self.monotonic_id_start = 0
# Keep IDs increasing so handles issued before the clear (e.g. for
# requests still draining) fail the ID check once their chunk is
# reused, instead of resolving to a different object.
self.monotonic_id_start = self.monotonic_id_end
self.data_buffer_start = 0
self.data_buffer_end = 0
def get_auth_secret(self) -> bytes:
"""Return the current handle-signing secret."""
assert self.shared_memory.buf is not None, "Buffer has been closed"
return bytes(self.shared_memory.buf[: self.AUTH_SECRET_BYTES])
def _init_auth_secret(self) -> None:
assert self.is_writer, "Only the writer can set the auth secret."
assert self.shared_memory.buf is not None, "Buffer has been closed"
self.shared_memory.buf[: self.AUTH_SECRET_BYTES] = secrets.token_bytes(
self.AUTH_SECRET_BYTES
)
def close(self) -> None:
"""Close the shared memory."""
if hasattr(self, "shared_memory"):
@@ -219,13 +246,14 @@ class SingleWriterShmRingBuffer:
# first 4 bytes as the monotonic id
buf_idx = self.data_buffer_end % self.data_buffer_size
self.shared_memory.buf[buf_idx : buf_idx + self.ID_NBYTES] = self.int2byte(
self.monotonic_id_end
physical_idx = self.DATA_OFFSET + buf_idx
self.shared_memory.buf[physical_idx : physical_idx + self.ID_NBYTES] = (
self.int2byte(self.monotonic_id_end)
)
# next 4 bytes as the size of the data buffer
self.shared_memory.buf[buf_idx + self.ID_NBYTES : buf_idx + self.MD_SIZE] = (
self.int2byte(size)
)
self.shared_memory.buf[
physical_idx + self.ID_NBYTES : physical_idx + self.MD_SIZE
] = self.int2byte(size)
# record metadata
self.metadata[self.monotonic_id_end % self.ID_MAX] = self.data_buffer_end
@@ -240,14 +268,19 @@ class SingleWriterShmRingBuffer:
def access_buf(self, address: int):
assert self.shared_memory.buf is not None, "Buffer has been closed"
buf_idx = address % self.data_buffer_size
physical_idx = self.DATA_OFFSET + buf_idx
# read metadata
metadata_buff = self.shared_memory.buf[buf_idx : buf_idx + self.MD_SIZE]
metadata_buff = self.shared_memory.buf[
physical_idx : physical_idx + self.MD_SIZE
]
id = self.byte2int(metadata_buff[: self.ID_NBYTES])
size = self.byte2int(metadata_buff[self.ID_NBYTES : self.MD_SIZE])
# yield the data buffer and metadata
data_buff = self.shared_memory.buf[buf_idx + self.MD_SIZE : buf_idx + size]
data_buff = self.shared_memory.buf[
physical_idx + self.MD_SIZE : physical_idx + size
]
with (
memoryview(data_buff) as data_view,
):
@@ -423,7 +456,8 @@ class SingleWriterShmObjectStorage:
Architecture:
- Single writer process can put(key, value) objects
- Multiple reader processes can get(address, monotonic_id) objects
- Multiple reader processes can get(address, monotonic_id, signature, key)
objects
- Built on SingleWriterShmRingBuffer for efficient shared memory management
- Thread-safe operations with reader synchronization via locks
@@ -439,6 +473,13 @@ class SingleWriterShmObjectStorage:
- Automatic Cleanup: Garbage collection happens transparently during
allocation
Handle Signatures:
- Handles are signed with an HMAC of (key, address, monotonic_id), keyed by
a random secret stored at the start of the shared memory
- The writer signs the handles it issues and rejects stale or forged
handles supplied by requests
- Readers verify the signature in get/touch before reading shared memory
Memory Layout per Object:
`[4-byte reference_count][metadata_size][serialized_object_data]`
@@ -521,6 +562,76 @@ class SingleWriterShmObjectStorage:
else:
raise ValueError(f"Unsupported data type for serialization: {type(data)}")
def _validate_monotonic_id(
self,
address: int,
monotonic_id: int,
buf_metadata: tuple[int, int],
) -> None:
if buf_metadata[0] != monotonic_id:
raise ValueError(
f"Data for address:id '{address}:{monotonic_id}'"
" has been modified or is invalid."
)
def _verify_reader_signature(
self,
address: int,
monotonic_id: int,
signature: list[int] | None,
key: str | None,
) -> None:
if signature is None or key is None:
raise ValueError("Missing SHM handle signature for cache key.")
if (
not isinstance(signature, list)
or len(signature) != hashlib.sha256().digest_size
or any(type(byte) is not int or not 0 <= byte <= 255 for byte in signature)
or not isinstance(key, str)
):
raise ValueError("Invalid SHM handle signature for cache key.")
expected_signature = self._make_signature(key, address, monotonic_id)
if not hmac.compare_digest(bytes(signature), expected_signature):
raise ValueError("Invalid SHM handle signature for cache key.")
def verify_signature(
self,
key: str | None,
address: int,
monotonic_id: int,
signature: list[int] | None,
) -> None:
"""Verify that a handle was issued for the given key.
For writers: the handle must also be the key's current entry
For readers: only the signature is checked
Args:
key: String key the handle belongs to
address: Address of the object
monotonic_id: Monotonic ID of the object
signature: Signature issued with the handle
"""
if self.is_writer and (
key is None or self.key_index.get(key) != (address, monotonic_id)
):
raise ValueError("Invalid SHM handle signature for cache key.")
self._verify_reader_signature(address, monotonic_id, signature, key)
def _make_signature(self, key: str, address: int, monotonic_id: int) -> bytes:
payload = (
hashlib.sha256(key.encode("utf-8")).digest()
+ address.to_bytes(8, "little", signed=False)
+ monotonic_id.to_bytes(4, "little", signed=False)
)
return hmac.new(
self.ring_buffer.get_auth_secret(),
payload,
hashlib.sha256,
).digest()
def increment_writer_flag(self, id: int) -> None:
"""Set the in-use flag for the writer."""
self.writer_flag[id] = self.writer_flag.get(id, 0) + 1
@@ -555,6 +666,11 @@ class SingleWriterShmObjectStorage:
self.increment_writer_flag(monotonic_id)
return address, monotonic_id
def get_signature(self, key: str) -> list[int]:
"""Sign the handle of a cached object so readers can verify it."""
address, monotonic_id = self.key_index[key]
return list(self._make_signature(key, address, monotonic_id))
def put(self, key: str, value: Any) -> tuple[int, int]:
"""Store a key-value pair in the object storage.
Attempts to free max_object_size bytes using FIFO order
@@ -605,15 +721,20 @@ class SingleWriterShmObjectStorage:
self.id_index[monotonic_id] = key
return address, monotonic_id
def get(self, address: int, monotonic_id: int) -> Any:
def get(
self,
address: int,
monotonic_id: int,
signature: list[int] | None = None,
key: str | None = None,
) -> Any:
if not self.is_writer:
# Reject forged handles before dereferencing the supplied address.
self.verify_signature(key, address, monotonic_id, signature)
# Read data from buffer
with self.ring_buffer.access_buf(address) as (data_view, buf_metadata):
# check id from metadata
if buf_metadata[0] != monotonic_id:
raise ValueError(
f"Data for address:id '{address}:{monotonic_id}'"
" has been modified or is invalid."
)
self._validate_monotonic_id(address, monotonic_id, buf_metadata)
obj = self.ser_de.deserialize(data_view[self.flag_bytes :])
@@ -633,6 +754,7 @@ class SingleWriterShmObjectStorage:
key: str,
address: int = 0,
monotonic_id: int = 0,
signature: list[int] | None = None,
) -> None:
"""Touch an existing cached item to update its eviction status.
@@ -643,6 +765,7 @@ class SingleWriterShmObjectStorage:
key: String key of the object to touch
address: Address of the object (only for readers)
monotonic_id: Monotonic ID of the object (only for readers)
signature: Server-issued handle signature (only for readers)
"""
if self._reader_lock is None:
@@ -652,10 +775,13 @@ class SingleWriterShmObjectStorage:
# Writer side: increment writer_flag to raise eviction threshold
self.increment_writer_flag(monotonic_id)
else:
# Reject forged handles before dereferencing the supplied address.
self.verify_signature(key, address, monotonic_id, signature)
with (
self._reader_lock,
self.ring_buffer.access_buf(address) as (data_view, _),
self.ring_buffer.access_buf(address) as (data_view, buf_metadata),
):
self._validate_monotonic_id(address, monotonic_id, buf_metadata)
reader_count = self.ring_buffer.byte2int(data_view[: self.flag_bytes])
# NOTE(Long):
@@ -39,6 +39,7 @@ from vllm.inputs import EngineInput, TokensPrompt, mm_input
from vllm.logger import init_logger
from vllm.logprobs import Logprob
from vllm.multimodal.inputs import (
MultiModalKwargsItem,
MultiModalKwargsItems,
PlaceholderRange,
)
@@ -106,6 +107,23 @@ class ServingTokens(GenerateBaseServing):
else getattr(mc, "override_generation_config", {}).get("max_new_tokens")
)
def _validate_mm_cache_handles(
self,
mm_kwargs: dict[str, list[MultiModalKwargsItem | None]],
mm_hashes: dict[str, list[str]],
) -> ErrorResponse | None:
cache = self.online_renderer.renderer.mm_processor_cache
if cache is None:
return None
try:
for modality, items in mm_kwargs.items():
for mm_hash, item in zip(mm_hashes[modality], items, strict=True):
if item is not None:
cache.validate_input_item(item, mm_hash)
except ValueError as error:
return self.create_error_response(error)
return None
async def serve_tokens(
self,
request: GenerateRequest,
@@ -201,6 +219,8 @@ class ServingTokens(GenerateBaseServing):
# Deserialize full tensor data and optional metadata-only data.
# Metadata-only items are valid when ec_transfer_params is set.
mm_kwargs = mm_kwargs_from_features(features)
if error := self._validate_mm_cache_handles(mm_kwargs, features.mm_hashes):
return error
engine_input = mm_input(
prompt_token_ids=request.token_ids,
+8
View File
@@ -356,6 +356,14 @@ class BaseMultiModalProcessorCache(
"""Close the underlying cache, if needed."""
pass
def validate_input_item(
self,
mm_item: MultiModalKwargsItem,
mm_hash: str,
) -> None:
"""Validate externally supplied cache metadata before engine handoff."""
return None
@abstractmethod
def touch_sender_cache_item(self, mm_hash: str) -> None:
"""Update the cache eviction order for a multi-modal item.
+73 -12
View File
@@ -4,7 +4,7 @@
from collections.abc import Sequence
from multiprocessing.synchronize import Lock as LockType
from typing import TYPE_CHECKING, cast
from typing import TYPE_CHECKING, TypeAlias
from typing_extensions import override
@@ -37,6 +37,29 @@ if TYPE_CHECKING:
logger = init_logger(__name__)
ShmHandle: TypeAlias = tuple[int, int, list[int]]
def _get_shm_handle(mm_item: MultiModalKwargsItem) -> ShmHandle | None:
if "address" not in mm_item:
return None
if "monotonic_id" not in mm_item or "signature" not in mm_item:
raise ValueError("Missing SHM handle signature for cache key.")
address = mm_item["address"].data
monotonic_id = mm_item["monotonic_id"].data
signature = mm_item["signature"].data
if (
type(address) is not int
or not 0 <= address < 2**64
or type(monotonic_id) is not int
or not 0 <= monotonic_id < 2**32
or not isinstance(signature, list)
):
raise ValueError("Invalid SHM handle signature for cache key.")
return address, monotonic_id, signature
class ShmObjectStoreSenderCache(BaseMultiModalProcessorCache):
"""The cache which is used on P0 when SHM caching is enabled.
@@ -98,8 +121,12 @@ class ShmObjectStoreSenderCache(BaseMultiModalProcessorCache):
self._total += 1
address, monotonic_id = self._shm_cache.get_cached(mm_hash)
signature = self._shm_cache.get_signature(mm_hash)
prompt_updates = self._p0_cache[mm_hash]
return self.address_as_item(address, monotonic_id), prompt_updates
return (
self.address_as_item(address, monotonic_id, signature),
prompt_updates,
)
assert mm_item is not None, f"Expected a cached item for {mm_hash=}"
item, prompt_updates = mm_item
@@ -108,12 +135,16 @@ class ShmObjectStoreSenderCache(BaseMultiModalProcessorCache):
try:
address, monotonic_id = self._shm_cache.put(mm_hash, item)
signature = self._shm_cache.get_signature(mm_hash)
# Try to remove dangling items if p0 cache is too large.
if len(self._p0_cache) >= 2 * len(self._shm_cache.key_index):
self.remove_dangling_items()
self._p0_cache[mm_hash] = prompt_updates
return self.address_as_item(address, monotonic_id), prompt_updates
return (
self.address_as_item(address, monotonic_id, signature),
prompt_updates,
)
except ValueError as e:
# `put` raises ValueError either for an oversize item or for a
# duplicate key (concurrent insert); the latter is benign so we
@@ -143,6 +174,21 @@ class ShmObjectStoreSenderCache(BaseMultiModalProcessorCache):
Increments writer_flag on sender side."""
self._shm_cache.touch(mm_hash)
@override
def validate_input_item(
self,
mm_item: MultiModalKwargsItem,
mm_hash: str,
) -> None:
if (handle := _get_shm_handle(mm_item)) is not None:
address, monotonic_id, signature = handle
self._shm_cache.verify_signature(
mm_hash,
address,
monotonic_id,
signature,
)
@override
def clear_cache(self) -> None:
self._shm_cache.clear()
@@ -171,6 +217,7 @@ class ShmObjectStoreSenderCache(BaseMultiModalProcessorCache):
self,
address: int,
monotonic_id: int,
signature: list[int],
) -> MultiModalKwargsItem:
addr_elem = MultiModalFieldElem(
data=address,
@@ -181,7 +228,18 @@ class ShmObjectStoreSenderCache(BaseMultiModalProcessorCache):
field=MultiModalBatchedField(),
)
return MultiModalKwargsItem({"address": addr_elem, "monotonic_id": id_elem})
signature_elem = MultiModalFieldElem(
data=signature,
field=MultiModalBatchedField(),
)
return MultiModalKwargsItem(
{
"address": addr_elem,
"monotonic_id": id_elem,
"signature": signature_elem,
}
)
class ShmObjectStoreReceiverCache(BaseMultiModalReceiverCache):
@@ -235,10 +293,9 @@ class ShmObjectStoreReceiverCache(BaseMultiModalReceiverCache):
mm_hash: str,
) -> MultiModalKwargsItem:
assert mm_item is not None, f"Expected an address item for {mm_hash=}"
if "address" in mm_item:
address = cast(int, mm_item["address"].data)
monotonic_id = cast(int, mm_item["monotonic_id"].data)
return self._shm_cache.get(address, monotonic_id)
if (handle := _get_shm_handle(mm_item)) is not None:
address, monotonic_id, signature = handle
return self._shm_cache.get(address, monotonic_id, signature, mm_hash)
return mm_item
@@ -251,10 +308,14 @@ class ShmObjectStoreReceiverCache(BaseMultiModalReceiverCache):
"""Touch the item in shared memory cache to prevent eviction.
Increments reader_count on receiver side."""
assert mm_item is not None
if "address" in mm_item:
address = cast(int, mm_item["address"].data)
monotonic_id = cast(int, mm_item["monotonic_id"].data)
self._shm_cache.touch(mm_hash, address=address, monotonic_id=monotonic_id)
if (handle := _get_shm_handle(mm_item)) is not None:
address, monotonic_id, signature = handle
self._shm_cache.touch(
mm_hash,
address=address,
monotonic_id=monotonic_id,
signature=signature,
)
@override
def clear_cache(self) -> None: