From 3d5625cc3c2474994107047d7a6ecb8e4008f9b0 Mon Sep 17 00:00:00 2001 From: Clinton Thomas <1033162+KernelClint@users.noreply.github.com> Date: Wed, 30 Sep 2026 08:16:26 -0400 Subject: [PATCH] [Security] Authenticate shared-memory multimodal cache handles (#59357) Signed-off-by: Clinton Thomas <1033162+KernelClint@users.noreply.github.com> Co-authored-by: Lucas Bourtoule <35483370+dhalf@users.noreply.github.com> Co-authored-by: Cyrus Leung --- tests/distributed/test_shm_buffer.py | 6 +- tests/distributed/test_shm_storage.py | 69 +++++- .../test_generate_stream.py | 53 ++++- tests/multimodal/test_cache.py | 208 ++++++++++++++++++ .../shm_object_storage.py | 166 ++++++++++++-- .../scale_out/token_in_token_out/serving.py | 20 ++ vllm/multimodal/cache/base.py | 8 + vllm/multimodal/cache/shm.py | 85 ++++++- 8 files changed, 574 insertions(+), 41 deletions(-) diff --git a/tests/distributed/test_shm_buffer.py b/tests/distributed/test_shm_buffer.py index 6f61c28322..7a919124db 100644 --- a/tests/distributed/test_shm_buffer.py +++ b/tests/distributed/test_shm_buffer.py @@ -108,10 +108,10 @@ class TestSingleWriterShmRingBuffer(unittest.TestCase): # Clear the buffer self.ring_buffer.clear() - # Check that metadata is empty and IDs reset + # Check that metadata is empty and IDs keep increasing self.assertEqual(len(self.ring_buffer.metadata), 0) - self.assertEqual(self.ring_buffer.monotonic_id_start, 0) - self.assertEqual(self.ring_buffer.monotonic_id_end, 0) + self.assertEqual(self.ring_buffer.monotonic_id_start, 3) + self.assertEqual(self.ring_buffer.monotonic_id_end, 3) self.assertEqual(self.ring_buffer.data_buffer_start, 0) self.assertEqual(self.ring_buffer.data_buffer_end, 0) diff --git a/tests/distributed/test_shm_storage.py b/tests/distributed/test_shm_storage.py index fb7d5528c0..762ffad8e6 100644 --- a/tests/distributed/test_shm_storage.py +++ b/tests/distributed/test_shm_storage.py @@ -227,10 +227,68 @@ class TestSingleWriterShmObjectStorage(unittest.TestCase): self.assertEqual(len(self.storage.id_index), 0) self.assertEqual(len(self.storage.ring_buffer.metadata), 0) - # Verify that new items can be added after clearing + # Verify that new items can be added after clearing. IDs keep + # increasing so handles issued before the clear cannot alias new data. address, monotonic_id = self.storage.put("new_item", "new_value") self.assertIn("new_item", self.storage.key_index) - self.assertEqual((address, monotonic_id), (0, 0)) + self.assertEqual((address, monotonic_id), (0, 5)) + + def _open_reader(self) -> SingleWriterShmObjectStorage: + reader = SingleWriterShmObjectStorage.create_from_handle(self.storage.handle()) + self.addCleanup(reader.close) + return reader + + def test_reader_get_and_touch_each_require_valid_signature(self): + address, monotonic_id = self.storage.put("key", "value") + signature = self.storage.get_signature("key") + reader = self._open_reader() + + reader.touch("key", address, monotonic_id, signature) + self.assertEqual(reader.get(address, monotonic_id, signature, "key"), "value") + + flipped = [signature[0] ^ 1, *signature[1:]] + for bad in (None, flipped, [signature], signature[:-1], [256] * 32): + with self.assertRaisesRegex(ValueError, "SHM handle signature"): + reader.get(address, monotonic_id, bad, "key") + with self.assertRaisesRegex(ValueError, "SHM handle signature"): + reader.touch("key", address, monotonic_id, bad) + + def test_signature_is_bound_to_key_and_address(self): + address, monotonic_id = self.storage.put("key", "value") + signature = self.storage.get_signature("key") + reader = self._open_reader() + + with self.assertRaisesRegex(ValueError, "SHM handle signature"): + reader.get(address, monotonic_id, signature, "other_key") + with self.assertRaisesRegex(ValueError, "SHM handle signature"): + reader.get(address + 1, monotonic_id, signature, "key") + + def test_writer_rejects_handle_no_longer_in_key_index(self): + address, monotonic_id = self.storage.put("key", "value") + signature = self.storage.get_signature("key") + self.storage.verify_signature("key", address, monotonic_id, signature) + + self.storage.clear() + + with self.assertRaisesRegex(ValueError, "SHM handle signature"): + self.storage.verify_signature("key", address, monotonic_id, signature) + + def test_handle_issued_before_clear_reads_own_data_until_reused(self): + address, monotonic_id = self.storage.put("key", "value") + signature = self.storage.get_signature("key") + reader = self._open_reader() + + self.storage.clear() + + # Requests still draining after a clear keep reading their own data. + self.assertEqual(reader.get(address, monotonic_id, signature, "key"), "value") + + self.storage.put("key", "new_value") + + with self.assertRaisesRegex(ValueError, "modified or is invalid"): + reader.get(address, monotonic_id, signature, "key") + with self.assertRaisesRegex(ValueError, "modified or is invalid"): + reader.touch("key", address, monotonic_id, signature) # Reader process function @@ -242,11 +300,11 @@ def reader_process(process_id, storage_handle, items_to_read): errors = [] - for key, original_value, address, monotonic_id in items_to_read: + for key, original_value, address, monotonic_id, signature in items_to_read: time.sleep(random.random() / 100) try: # Read data from shared memory - retrieved_value = reader_storage.get(address, monotonic_id) + retrieved_value = reader_storage.get(address, monotonic_id, signature, key) # Verify data integrity assert retrieved_value == original_value @@ -289,7 +347,8 @@ def run_multiprocess_example(): for key, value in test_data: print(f"Storing {key}: {value}") address, monotonic_id = storage.put(key, value) - stored_items.append((key, value, address, monotonic_id)) + signature = storage.get_signature(key) + stored_items.append((key, value, address, monotonic_id, signature)) print(f" -> Stored at address {address}, ID {monotonic_id}") print("\n--- Retrieving Data ---") diff --git a/tests/entrypoints/scale_out/token_in_token_out/test_generate_stream.py b/tests/entrypoints/scale_out/token_in_token_out/test_generate_stream.py index f7c6f05e7f..28f8a505f8 100644 --- a/tests/entrypoints/scale_out/token_in_token_out/test_generate_stream.py +++ b/tests/entrypoints/scale_out/token_in_token_out/test_generate_stream.py @@ -15,14 +15,25 @@ from vllm.entrypoints.openai.models.serving import OpenAIServingModels from vllm.entrypoints.scale_out.token_in_token_out.mm_features import ( placeholder_ranges_from_engine_input, ) +from vllm.entrypoints.scale_out.token_in_token_out.mm_serde import ( + encode_mm_kwargs_item, +) from vllm.entrypoints.scale_out.token_in_token_out.protocol import ( GenerateRequest, GenerateResponse, + MultiModalFeatures, + PlaceholderRangeInfo, ) from vllm.entrypoints.scale_out.token_in_token_out.serving import ServingTokens +from vllm.entrypoints.serve.engine.protocol import ErrorResponse from vllm.exceptions import GenerationError from vllm.logprobs import Logprob -from vllm.multimodal.inputs import PlaceholderRange +from vllm.multimodal.inputs import ( + MultiModalBatchedField, + MultiModalFieldElem, + MultiModalKwargsItem, + PlaceholderRange, +) from vllm.outputs import CompletionOutput, RequestOutput from vllm.renderers import renderer_from_config from vllm.renderers.online_renderer import OnlineRenderer @@ -285,6 +296,46 @@ async def test_non_stream_error(): await serving.serve_tokens(request) +@pytest.mark.asyncio +async def test_serve_tokens_rejects_invalid_shm_handle_before_engine_handoff(): + engine = _mock_engine() + engine.generate = MagicMock() + serving = _build_serving_tokens(engine) + cache = MagicMock() + cache.validate_input_item.side_effect = ValueError( + "Invalid SHM handle signature for cache key." + ) + serving.online_renderer.renderer._mm_processor_cache = cache + forged_item = MultiModalKwargsItem( + { + "address": MultiModalFieldElem(data=224, field=MultiModalBatchedField()), + "monotonic_id": MultiModalFieldElem( + data=4242, + field=MultiModalBatchedField(), + ), + } + ) + request = GenerateRequest( + token_ids=[1, 2, 3], + sampling_params=SamplingParams(max_tokens=1), + model=MODEL_NAME, + features=MultiModalFeatures( + mm_hashes={"image": ["image_A"]}, + mm_placeholders={"image": [PlaceholderRangeInfo(offset=0, length=1)]}, + kwargs_data={"image": [encode_mm_kwargs_item(forged_item)]}, + ), + ) + + response = await serving.serve_tokens(request) + + assert isinstance(response, ErrorResponse) + cache.validate_input_item.assert_called_once() + validated_item, validated_hash = cache.validate_input_item.call_args.args + assert validated_hash == "image_A" + assert validated_item["address"].data == 224 + engine.generate.assert_not_called() + + @pytest.mark.asyncio async def test_stream_basic(): """Streaming returns SSE chunks with correct token_ids and ends with [DONE].""" diff --git a/tests/multimodal/test_cache.py b/tests/multimodal/test_cache.py index a4966565f6..0416734c24 100644 --- a/tests/multimodal/test_cache.py +++ b/tests/multimodal/test_cache.py @@ -29,6 +29,7 @@ from vllm.multimodal.cache.base import ( ) from vllm.multimodal.hasher import MultiModalHasher from vllm.multimodal.inputs import ( + MultiModalBatchedField, MultiModalFeatureSpec, MultiModalFieldElem, MultiModalKwargsItem, @@ -42,6 +43,7 @@ from vllm.utils.async_utils import make_async from vllm.utils.mem_constants import GiB_bytes, MiB_bytes from vllm.v1.engine.async_llm import AsyncLLM from vllm.v1.engine.llm_engine import LLMEngine +from vllm.v1.serial_utils import MsgpackDecoder, MsgpackEncoder from ..models.utils import build_model_context from ..utils import create_new_process_for_each_test @@ -778,6 +780,212 @@ def test_cache_eviction_shm_cache(): _run_test_cache_eviction_shm(sender_cache, receiver_cache, base_item_size=MiB_bytes) +def test_shm_receiver_requires_sender_handle_for_current_cache_key(): + vllm_config = VllmConfig( + model_config=ModelConfig( + model="llava-hf/llava-onevision-qwen2-0.5b-ov-hf", + mm_processor_cache_type="shm", + mm_shm_cache_max_object_size_mb=1, + mm_processor_cache_gb=2 * MiB_bytes / GiB_bytes, + ), + ) + sender_cache = ShmObjectStoreSenderCache(vllm_config) + receiver_cache = ShmObjectStoreReceiverCache(vllm_config, mp.Lock()) + mm_hash = "image_A" + item = MultiModalKwargsItem.dummy(1024) + + try: + sender_item, _ = sender_cache.get_and_update_item((item, []), mm_hash) + + valid_feature = MultiModalFeatureSpec( + data=sender_item, + modality="image", + identifier=mm_hash, + mm_position=PlaceholderRange(offset=0, length=1), + mm_hash=mm_hash, + ) + receiver_cache.get_and_update_features([valid_feature]) + assert valid_feature.data == item + + forged_item = MultiModalKwargsItem( + { + "address": sender_item["address"], + "monotonic_id": sender_item["monotonic_id"], + } + ) + forged_feature = MultiModalFeatureSpec( + data=forged_item, + modality="image", + identifier=mm_hash, + mm_position=PlaceholderRange(offset=0, length=1), + mm_hash=mm_hash, + ) + with pytest.raises(ValueError, match="SHM handle signature"): + receiver_cache.get_and_update_features([forged_feature]) + + wrong_key = "image_B" + wrong_key_feature = MultiModalFeatureSpec( + data=sender_item, + modality="image", + identifier=wrong_key, + mm_position=PlaceholderRange(offset=0, length=1), + mm_hash=wrong_key, + ) + with pytest.raises(ValueError, match="SHM handle signature"): + receiver_cache.get_and_update_features([wrong_key_feature]) + finally: + receiver_cache._shm_cache.close() + sender_cache.close() + + +def test_shm_sender_handle_survives_engine_msgpack_round_trip(): + vllm_config = VllmConfig( + model_config=ModelConfig( + model="llava-hf/llava-onevision-qwen2-0.5b-ov-hf", + mm_processor_cache_type="shm", + mm_shm_cache_max_object_size_mb=1, + mm_processor_cache_gb=2 * MiB_bytes / GiB_bytes, + ), + ) + sender_cache = ShmObjectStoreSenderCache(vllm_config) + receiver_cache = ShmObjectStoreReceiverCache(vllm_config, mp.Lock()) + mm_hash = "image_A" + item = MultiModalKwargsItem.dummy(1024) + + try: + sender_item, _ = sender_cache.get_and_update_item((item, []), mm_hash) + encoded = MsgpackEncoder().encode(sender_item) + decoded = MsgpackDecoder(MultiModalKwargsItem).decode(encoded) + feature = MultiModalFeatureSpec( + data=decoded, + modality="image", + identifier=mm_hash, + mm_position=PlaceholderRange(offset=0, length=1), + mm_hash=mm_hash, + ) + + receiver_cache.get_and_update_features([feature]) + + assert feature.data == item + finally: + receiver_cache._shm_cache.close() + sender_cache.close() + + +def _small_shm_vllm_config() -> VllmConfig: + return VllmConfig( + model_config=ModelConfig( + model="llava-hf/llava-onevision-qwen2-0.5b-ov-hf", + mm_processor_cache_type="shm", + mm_shm_cache_max_object_size_mb=1, + mm_processor_cache_gb=2 * MiB_bytes / GiB_bytes, + ), + ) + + +def _with_elem(item: MultiModalKwargsItem, key: str, data) -> MultiModalKwargsItem: + elems = dict(item) + elems[key] = MultiModalFieldElem(data=data, field=MultiModalBatchedField()) + return MultiModalKwargsItem(elems) + + +def test_shm_sender_validates_client_supplied_handles(): + sender_cache = ShmObjectStoreSenderCache(_small_shm_vllm_config()) + mm_hash = "image_A" + + try: + handle, _ = sender_cache.get_and_update_item( + (MultiModalKwargsItem.dummy(1024), []), mm_hash + ) + sender_cache.validate_input_item(handle, mm_hash) + sender_cache.validate_input_item(MultiModalKwargsItem.dummy(16), mm_hash) + + signature = handle["signature"].data + forged = [ + _with_elem(handle, "signature", [signature[0] ^ 1, *signature[1:]]), + _with_elem(handle, "signature", [signature]), + _with_elem(handle, "address", True), + _with_elem(handle, "address", -1), + _with_elem(handle, "monotonic_id", 2**32), + MultiModalKwargsItem( + {"address": handle["address"], "monotonic_id": handle["monotonic_id"]} + ), + ] + for item in forged: + with pytest.raises(ValueError, match="SHM handle signature"): + sender_cache.validate_input_item(item, mm_hash) + with pytest.raises(ValueError, match="SHM handle signature"): + sender_cache.validate_input_item(handle, "image_B") + + sender_cache.clear_cache() + with pytest.raises(ValueError, match="SHM handle signature"): + sender_cache.validate_input_item(handle, mm_hash) + finally: + sender_cache.close() + + +def test_shm_receiver_rejects_out_of_range_handle_fields(): + vllm_config = _small_shm_vllm_config() + sender_cache = ShmObjectStoreSenderCache(vllm_config) + receiver_cache = ShmObjectStoreReceiverCache(vllm_config, mp.Lock()) + mm_hash = "image_A" + + try: + handle, _ = sender_cache.get_and_update_item( + (MultiModalKwargsItem.dummy(1024), []), mm_hash + ) + for key, value in [ + ("address", -1), + ("address", 2**64), + ("monotonic_id", -1), + ("monotonic_id", 2**32), + ]: + with pytest.raises(ValueError, match="SHM handle signature"): + receiver_cache.get_and_update_item( + _with_elem(handle, key, value), mm_hash + ) + finally: + receiver_cache._shm_cache.close() + sender_cache.close() + + +def test_shm_handle_issued_before_cache_clear(): + vllm_config = _small_shm_vllm_config() + sender_cache = ShmObjectStoreSenderCache(vllm_config) + receiver_cache = ShmObjectStoreReceiverCache(vllm_config, mp.Lock()) + mm_hash = "image_A" + item = MultiModalKwargsItem.dummy(1024) + + def feature(data: MultiModalKwargsItem) -> MultiModalFeatureSpec: + return MultiModalFeatureSpec( + data=data, + modality="image", + identifier=mm_hash, + mm_position=PlaceholderRange(offset=0, length=1), + mm_hash=mm_hash, + ) + + try: + stale_item, _ = sender_cache.get_and_update_item((item, []), mm_hash) + sender_cache.clear_cache() + + # A request still draining after the clear resolves to its own data. + draining = feature(stale_item) + receiver_cache.get_and_update_features([draining]) + assert draining.data == item + + sender_cache.get_and_update_item( + (MultiModalKwargsItem.dummy(2048), []), + mm_hash, + ) + + with pytest.raises(ValueError, match="modified or is invalid"): + receiver_cache.get_and_update_features([feature(stale_item)]) + finally: + receiver_cache._shm_cache.close() + sender_cache.close() + + def test_processor_cache_shared_across_loras(): """Test that processor cache uses mm_hash to share data across LoRAs.""" model_config = ModelConfig( diff --git a/vllm/distributed/device_communicators/shm_object_storage.py b/vllm/distributed/device_communicators/shm_object_storage.py index bc4f1b6e51..3da8c5dcf6 100644 --- a/vllm/distributed/device_communicators/shm_object_storage.py +++ b/vllm/distributed/device_communicators/shm_object_storage.py @@ -1,7 +1,10 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import hashlib +import hmac import pickle +import secrets from abc import ABC, abstractmethod from collections.abc import Callable, Iterable from contextlib import contextmanager, suppress @@ -97,6 +100,9 @@ class SingleWriterShmRingBuffer: - Reader synchronization handled by is_free_fn callback - Writer handles garbage collection (free_buf) based on reader feedback + Shared Memory Layout: + `[32-byte handle-signing secret][ring buffer chunks...]` + Memory Layout per Buffer Chunk: `[4-byte monotonic_id][4-byte chunk_size][actual_data...]` ^metadata_start ^data_start @@ -119,6 +125,8 @@ class SingleWriterShmRingBuffer: self.SIZE_NBYTES = 4 # 4 bytes for id, 4 bytes for buffer size self.MD_SIZE = self.ID_NBYTES + self.SIZE_NBYTES + self.AUTH_SECRET_BYTES = 32 + self.DATA_OFFSET = self.AUTH_SECRET_BYTES self.monotonic_id_end = 0 self.monotonic_id_start = 0 self.data_buffer_start = 0 @@ -129,8 +137,11 @@ class SingleWriterShmRingBuffer: # we are creating a buffer self.metadata: dict[int, int] = {} # monotonic_id -> start address self.shared_memory = shared_memory.SharedMemory( - create=True, size=self.data_buffer_size, name=name + create=True, + size=self.data_buffer_size + self.DATA_OFFSET, + name=name, ) + self._init_auth_secret() else: # we are opening an existing buffer # fix to https://stackoverflow.com/q/62748654/9191338 @@ -146,7 +157,9 @@ class SingleWriterShmRingBuffer: # so the shared memory block size may be larger or equal # to the requested size. The size parameter is ignored # when attaching to an existing block. - assert self.shared_memory.size >= self.data_buffer_size + assert ( + self.shared_memory.size >= self.data_buffer_size + self.DATA_OFFSET + ) logger.debug( "Shared memory created/opened with name: %s, size: %d", @@ -164,11 +177,25 @@ class SingleWriterShmRingBuffer: """Clear the ring buffer.""" assert self.is_writer, "Only the writer can clear the buffer." self.metadata.clear() - self.monotonic_id_end = 0 - self.monotonic_id_start = 0 + # Keep IDs increasing so handles issued before the clear (e.g. for + # requests still draining) fail the ID check once their chunk is + # reused, instead of resolving to a different object. + self.monotonic_id_start = self.monotonic_id_end self.data_buffer_start = 0 self.data_buffer_end = 0 + def get_auth_secret(self) -> bytes: + """Return the current handle-signing secret.""" + assert self.shared_memory.buf is not None, "Buffer has been closed" + return bytes(self.shared_memory.buf[: self.AUTH_SECRET_BYTES]) + + def _init_auth_secret(self) -> None: + assert self.is_writer, "Only the writer can set the auth secret." + assert self.shared_memory.buf is not None, "Buffer has been closed" + self.shared_memory.buf[: self.AUTH_SECRET_BYTES] = secrets.token_bytes( + self.AUTH_SECRET_BYTES + ) + def close(self) -> None: """Close the shared memory.""" if hasattr(self, "shared_memory"): @@ -219,13 +246,14 @@ class SingleWriterShmRingBuffer: # first 4 bytes as the monotonic id buf_idx = self.data_buffer_end % self.data_buffer_size - self.shared_memory.buf[buf_idx : buf_idx + self.ID_NBYTES] = self.int2byte( - self.monotonic_id_end + physical_idx = self.DATA_OFFSET + buf_idx + self.shared_memory.buf[physical_idx : physical_idx + self.ID_NBYTES] = ( + self.int2byte(self.monotonic_id_end) ) # next 4 bytes as the size of the data buffer - self.shared_memory.buf[buf_idx + self.ID_NBYTES : buf_idx + self.MD_SIZE] = ( - self.int2byte(size) - ) + self.shared_memory.buf[ + physical_idx + self.ID_NBYTES : physical_idx + self.MD_SIZE + ] = self.int2byte(size) # record metadata self.metadata[self.monotonic_id_end % self.ID_MAX] = self.data_buffer_end @@ -240,14 +268,19 @@ class SingleWriterShmRingBuffer: def access_buf(self, address: int): assert self.shared_memory.buf is not None, "Buffer has been closed" buf_idx = address % self.data_buffer_size + physical_idx = self.DATA_OFFSET + buf_idx # read metadata - metadata_buff = self.shared_memory.buf[buf_idx : buf_idx + self.MD_SIZE] + metadata_buff = self.shared_memory.buf[ + physical_idx : physical_idx + self.MD_SIZE + ] id = self.byte2int(metadata_buff[: self.ID_NBYTES]) size = self.byte2int(metadata_buff[self.ID_NBYTES : self.MD_SIZE]) # yield the data buffer and metadata - data_buff = self.shared_memory.buf[buf_idx + self.MD_SIZE : buf_idx + size] + data_buff = self.shared_memory.buf[ + physical_idx + self.MD_SIZE : physical_idx + size + ] with ( memoryview(data_buff) as data_view, ): @@ -423,7 +456,8 @@ class SingleWriterShmObjectStorage: Architecture: - Single writer process can put(key, value) objects - - Multiple reader processes can get(address, monotonic_id) objects + - Multiple reader processes can get(address, monotonic_id, signature, key) + objects - Built on SingleWriterShmRingBuffer for efficient shared memory management - Thread-safe operations with reader synchronization via locks @@ -439,6 +473,13 @@ class SingleWriterShmObjectStorage: - Automatic Cleanup: Garbage collection happens transparently during allocation + Handle Signatures: + - Handles are signed with an HMAC of (key, address, monotonic_id), keyed by + a random secret stored at the start of the shared memory + - The writer signs the handles it issues and rejects stale or forged + handles supplied by requests + - Readers verify the signature in get/touch before reading shared memory + Memory Layout per Object: `[4-byte reference_count][metadata_size][serialized_object_data]` @@ -521,6 +562,76 @@ class SingleWriterShmObjectStorage: else: raise ValueError(f"Unsupported data type for serialization: {type(data)}") + def _validate_monotonic_id( + self, + address: int, + monotonic_id: int, + buf_metadata: tuple[int, int], + ) -> None: + if buf_metadata[0] != monotonic_id: + raise ValueError( + f"Data for address:id '{address}:{monotonic_id}'" + " has been modified or is invalid." + ) + + def _verify_reader_signature( + self, + address: int, + monotonic_id: int, + signature: list[int] | None, + key: str | None, + ) -> None: + if signature is None or key is None: + raise ValueError("Missing SHM handle signature for cache key.") + if ( + not isinstance(signature, list) + or len(signature) != hashlib.sha256().digest_size + or any(type(byte) is not int or not 0 <= byte <= 255 for byte in signature) + or not isinstance(key, str) + ): + raise ValueError("Invalid SHM handle signature for cache key.") + + expected_signature = self._make_signature(key, address, monotonic_id) + if not hmac.compare_digest(bytes(signature), expected_signature): + raise ValueError("Invalid SHM handle signature for cache key.") + + def verify_signature( + self, + key: str | None, + address: int, + monotonic_id: int, + signature: list[int] | None, + ) -> None: + """Verify that a handle was issued for the given key. + + For writers: the handle must also be the key's current entry + For readers: only the signature is checked + + Args: + key: String key the handle belongs to + address: Address of the object + monotonic_id: Monotonic ID of the object + signature: Signature issued with the handle + + """ + if self.is_writer and ( + key is None or self.key_index.get(key) != (address, monotonic_id) + ): + raise ValueError("Invalid SHM handle signature for cache key.") + self._verify_reader_signature(address, monotonic_id, signature, key) + + def _make_signature(self, key: str, address: int, monotonic_id: int) -> bytes: + payload = ( + hashlib.sha256(key.encode("utf-8")).digest() + + address.to_bytes(8, "little", signed=False) + + monotonic_id.to_bytes(4, "little", signed=False) + ) + return hmac.new( + self.ring_buffer.get_auth_secret(), + payload, + hashlib.sha256, + ).digest() + def increment_writer_flag(self, id: int) -> None: """Set the in-use flag for the writer.""" self.writer_flag[id] = self.writer_flag.get(id, 0) + 1 @@ -555,6 +666,11 @@ class SingleWriterShmObjectStorage: self.increment_writer_flag(monotonic_id) return address, monotonic_id + def get_signature(self, key: str) -> list[int]: + """Sign the handle of a cached object so readers can verify it.""" + address, monotonic_id = self.key_index[key] + return list(self._make_signature(key, address, monotonic_id)) + def put(self, key: str, value: Any) -> tuple[int, int]: """Store a key-value pair in the object storage. Attempts to free max_object_size bytes using FIFO order @@ -605,15 +721,20 @@ class SingleWriterShmObjectStorage: self.id_index[monotonic_id] = key return address, monotonic_id - def get(self, address: int, monotonic_id: int) -> Any: + def get( + self, + address: int, + monotonic_id: int, + signature: list[int] | None = None, + key: str | None = None, + ) -> Any: + if not self.is_writer: + # Reject forged handles before dereferencing the supplied address. + self.verify_signature(key, address, monotonic_id, signature) + # Read data from buffer with self.ring_buffer.access_buf(address) as (data_view, buf_metadata): - # check id from metadata - if buf_metadata[0] != monotonic_id: - raise ValueError( - f"Data for address:id '{address}:{monotonic_id}'" - " has been modified or is invalid." - ) + self._validate_monotonic_id(address, monotonic_id, buf_metadata) obj = self.ser_de.deserialize(data_view[self.flag_bytes :]) @@ -633,6 +754,7 @@ class SingleWriterShmObjectStorage: key: str, address: int = 0, monotonic_id: int = 0, + signature: list[int] | None = None, ) -> None: """Touch an existing cached item to update its eviction status. @@ -643,6 +765,7 @@ class SingleWriterShmObjectStorage: key: String key of the object to touch address: Address of the object (only for readers) monotonic_id: Monotonic ID of the object (only for readers) + signature: Server-issued handle signature (only for readers) """ if self._reader_lock is None: @@ -652,10 +775,13 @@ class SingleWriterShmObjectStorage: # Writer side: increment writer_flag to raise eviction threshold self.increment_writer_flag(monotonic_id) else: + # Reject forged handles before dereferencing the supplied address. + self.verify_signature(key, address, monotonic_id, signature) with ( self._reader_lock, - self.ring_buffer.access_buf(address) as (data_view, _), + self.ring_buffer.access_buf(address) as (data_view, buf_metadata), ): + self._validate_monotonic_id(address, monotonic_id, buf_metadata) reader_count = self.ring_buffer.byte2int(data_view[: self.flag_bytes]) # NOTE(Long): diff --git a/vllm/entrypoints/scale_out/token_in_token_out/serving.py b/vllm/entrypoints/scale_out/token_in_token_out/serving.py index 8b0fbff99b..ade103bbbc 100644 --- a/vllm/entrypoints/scale_out/token_in_token_out/serving.py +++ b/vllm/entrypoints/scale_out/token_in_token_out/serving.py @@ -39,6 +39,7 @@ from vllm.inputs import EngineInput, TokensPrompt, mm_input from vllm.logger import init_logger from vllm.logprobs import Logprob from vllm.multimodal.inputs import ( + MultiModalKwargsItem, MultiModalKwargsItems, PlaceholderRange, ) @@ -106,6 +107,23 @@ class ServingTokens(GenerateBaseServing): else getattr(mc, "override_generation_config", {}).get("max_new_tokens") ) + def _validate_mm_cache_handles( + self, + mm_kwargs: dict[str, list[MultiModalKwargsItem | None]], + mm_hashes: dict[str, list[str]], + ) -> ErrorResponse | None: + cache = self.online_renderer.renderer.mm_processor_cache + if cache is None: + return None + try: + for modality, items in mm_kwargs.items(): + for mm_hash, item in zip(mm_hashes[modality], items, strict=True): + if item is not None: + cache.validate_input_item(item, mm_hash) + except ValueError as error: + return self.create_error_response(error) + return None + async def serve_tokens( self, request: GenerateRequest, @@ -201,6 +219,8 @@ class ServingTokens(GenerateBaseServing): # Deserialize full tensor data and optional metadata-only data. # Metadata-only items are valid when ec_transfer_params is set. mm_kwargs = mm_kwargs_from_features(features) + if error := self._validate_mm_cache_handles(mm_kwargs, features.mm_hashes): + return error engine_input = mm_input( prompt_token_ids=request.token_ids, diff --git a/vllm/multimodal/cache/base.py b/vllm/multimodal/cache/base.py index 1b9b757917..d2aaeb50e3 100644 --- a/vllm/multimodal/cache/base.py +++ b/vllm/multimodal/cache/base.py @@ -356,6 +356,14 @@ class BaseMultiModalProcessorCache( """Close the underlying cache, if needed.""" pass + def validate_input_item( + self, + mm_item: MultiModalKwargsItem, + mm_hash: str, + ) -> None: + """Validate externally supplied cache metadata before engine handoff.""" + return None + @abstractmethod def touch_sender_cache_item(self, mm_hash: str) -> None: """Update the cache eviction order for a multi-modal item. diff --git a/vllm/multimodal/cache/shm.py b/vllm/multimodal/cache/shm.py index dddf3494cf..ab84dcf6af 100644 --- a/vllm/multimodal/cache/shm.py +++ b/vllm/multimodal/cache/shm.py @@ -4,7 +4,7 @@ from collections.abc import Sequence from multiprocessing.synchronize import Lock as LockType -from typing import TYPE_CHECKING, cast +from typing import TYPE_CHECKING, TypeAlias from typing_extensions import override @@ -37,6 +37,29 @@ if TYPE_CHECKING: logger = init_logger(__name__) +ShmHandle: TypeAlias = tuple[int, int, list[int]] + + +def _get_shm_handle(mm_item: MultiModalKwargsItem) -> ShmHandle | None: + if "address" not in mm_item: + return None + if "monotonic_id" not in mm_item or "signature" not in mm_item: + raise ValueError("Missing SHM handle signature for cache key.") + + address = mm_item["address"].data + monotonic_id = mm_item["monotonic_id"].data + signature = mm_item["signature"].data + if ( + type(address) is not int + or not 0 <= address < 2**64 + or type(monotonic_id) is not int + or not 0 <= monotonic_id < 2**32 + or not isinstance(signature, list) + ): + raise ValueError("Invalid SHM handle signature for cache key.") + + return address, monotonic_id, signature + class ShmObjectStoreSenderCache(BaseMultiModalProcessorCache): """The cache which is used on P0 when SHM caching is enabled. @@ -98,8 +121,12 @@ class ShmObjectStoreSenderCache(BaseMultiModalProcessorCache): self._total += 1 address, monotonic_id = self._shm_cache.get_cached(mm_hash) + signature = self._shm_cache.get_signature(mm_hash) prompt_updates = self._p0_cache[mm_hash] - return self.address_as_item(address, monotonic_id), prompt_updates + return ( + self.address_as_item(address, monotonic_id, signature), + prompt_updates, + ) assert mm_item is not None, f"Expected a cached item for {mm_hash=}" item, prompt_updates = mm_item @@ -108,12 +135,16 @@ class ShmObjectStoreSenderCache(BaseMultiModalProcessorCache): try: address, monotonic_id = self._shm_cache.put(mm_hash, item) + signature = self._shm_cache.get_signature(mm_hash) # Try to remove dangling items if p0 cache is too large. if len(self._p0_cache) >= 2 * len(self._shm_cache.key_index): self.remove_dangling_items() self._p0_cache[mm_hash] = prompt_updates - return self.address_as_item(address, monotonic_id), prompt_updates + return ( + self.address_as_item(address, monotonic_id, signature), + prompt_updates, + ) except ValueError as e: # `put` raises ValueError either for an oversize item or for a # duplicate key (concurrent insert); the latter is benign so we @@ -143,6 +174,21 @@ class ShmObjectStoreSenderCache(BaseMultiModalProcessorCache): Increments writer_flag on sender side.""" self._shm_cache.touch(mm_hash) + @override + def validate_input_item( + self, + mm_item: MultiModalKwargsItem, + mm_hash: str, + ) -> None: + if (handle := _get_shm_handle(mm_item)) is not None: + address, monotonic_id, signature = handle + self._shm_cache.verify_signature( + mm_hash, + address, + monotonic_id, + signature, + ) + @override def clear_cache(self) -> None: self._shm_cache.clear() @@ -171,6 +217,7 @@ class ShmObjectStoreSenderCache(BaseMultiModalProcessorCache): self, address: int, monotonic_id: int, + signature: list[int], ) -> MultiModalKwargsItem: addr_elem = MultiModalFieldElem( data=address, @@ -181,7 +228,18 @@ class ShmObjectStoreSenderCache(BaseMultiModalProcessorCache): field=MultiModalBatchedField(), ) - return MultiModalKwargsItem({"address": addr_elem, "monotonic_id": id_elem}) + signature_elem = MultiModalFieldElem( + data=signature, + field=MultiModalBatchedField(), + ) + + return MultiModalKwargsItem( + { + "address": addr_elem, + "monotonic_id": id_elem, + "signature": signature_elem, + } + ) class ShmObjectStoreReceiverCache(BaseMultiModalReceiverCache): @@ -235,10 +293,9 @@ class ShmObjectStoreReceiverCache(BaseMultiModalReceiverCache): mm_hash: str, ) -> MultiModalKwargsItem: assert mm_item is not None, f"Expected an address item for {mm_hash=}" - if "address" in mm_item: - address = cast(int, mm_item["address"].data) - monotonic_id = cast(int, mm_item["monotonic_id"].data) - return self._shm_cache.get(address, monotonic_id) + if (handle := _get_shm_handle(mm_item)) is not None: + address, monotonic_id, signature = handle + return self._shm_cache.get(address, monotonic_id, signature, mm_hash) return mm_item @@ -251,10 +308,14 @@ class ShmObjectStoreReceiverCache(BaseMultiModalReceiverCache): """Touch the item in shared memory cache to prevent eviction. Increments reader_count on receiver side.""" assert mm_item is not None - if "address" in mm_item: - address = cast(int, mm_item["address"].data) - monotonic_id = cast(int, mm_item["monotonic_id"].data) - self._shm_cache.touch(mm_hash, address=address, monotonic_id=monotonic_id) + if (handle := _get_shm_handle(mm_item)) is not None: + address, monotonic_id, signature = handle + self._shm_cache.touch( + mm_hash, + address=address, + monotonic_id=monotonic_id, + signature=signature, + ) @override def clear_cache(self) -> None: