[Security] Authenticate shared-memory multimodal cache handles (#59357)
Signed-off-by: Clinton Thomas <1033162+KernelClint@users.noreply.github.com> Co-authored-by: Lucas Bourtoule <35483370+dhalf@users.noreply.github.com> Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
This commit is contained in:
co-authored by
Lucas Bourtoule
Cyrus Leung
parent
72e7874fa6
commit
3d5625cc3c
@@ -108,10 +108,10 @@ class TestSingleWriterShmRingBuffer(unittest.TestCase):
|
||||
# Clear the buffer
|
||||
self.ring_buffer.clear()
|
||||
|
||||
# Check that metadata is empty and IDs reset
|
||||
# Check that metadata is empty and IDs keep increasing
|
||||
self.assertEqual(len(self.ring_buffer.metadata), 0)
|
||||
self.assertEqual(self.ring_buffer.monotonic_id_start, 0)
|
||||
self.assertEqual(self.ring_buffer.monotonic_id_end, 0)
|
||||
self.assertEqual(self.ring_buffer.monotonic_id_start, 3)
|
||||
self.assertEqual(self.ring_buffer.monotonic_id_end, 3)
|
||||
self.assertEqual(self.ring_buffer.data_buffer_start, 0)
|
||||
self.assertEqual(self.ring_buffer.data_buffer_end, 0)
|
||||
|
||||
|
||||
@@ -227,10 +227,68 @@ class TestSingleWriterShmObjectStorage(unittest.TestCase):
|
||||
self.assertEqual(len(self.storage.id_index), 0)
|
||||
self.assertEqual(len(self.storage.ring_buffer.metadata), 0)
|
||||
|
||||
# Verify that new items can be added after clearing
|
||||
# Verify that new items can be added after clearing. IDs keep
|
||||
# increasing so handles issued before the clear cannot alias new data.
|
||||
address, monotonic_id = self.storage.put("new_item", "new_value")
|
||||
self.assertIn("new_item", self.storage.key_index)
|
||||
self.assertEqual((address, monotonic_id), (0, 0))
|
||||
self.assertEqual((address, monotonic_id), (0, 5))
|
||||
|
||||
def _open_reader(self) -> SingleWriterShmObjectStorage:
|
||||
reader = SingleWriterShmObjectStorage.create_from_handle(self.storage.handle())
|
||||
self.addCleanup(reader.close)
|
||||
return reader
|
||||
|
||||
def test_reader_get_and_touch_each_require_valid_signature(self):
|
||||
address, monotonic_id = self.storage.put("key", "value")
|
||||
signature = self.storage.get_signature("key")
|
||||
reader = self._open_reader()
|
||||
|
||||
reader.touch("key", address, monotonic_id, signature)
|
||||
self.assertEqual(reader.get(address, monotonic_id, signature, "key"), "value")
|
||||
|
||||
flipped = [signature[0] ^ 1, *signature[1:]]
|
||||
for bad in (None, flipped, [signature], signature[:-1], [256] * 32):
|
||||
with self.assertRaisesRegex(ValueError, "SHM handle signature"):
|
||||
reader.get(address, monotonic_id, bad, "key")
|
||||
with self.assertRaisesRegex(ValueError, "SHM handle signature"):
|
||||
reader.touch("key", address, monotonic_id, bad)
|
||||
|
||||
def test_signature_is_bound_to_key_and_address(self):
|
||||
address, monotonic_id = self.storage.put("key", "value")
|
||||
signature = self.storage.get_signature("key")
|
||||
reader = self._open_reader()
|
||||
|
||||
with self.assertRaisesRegex(ValueError, "SHM handle signature"):
|
||||
reader.get(address, monotonic_id, signature, "other_key")
|
||||
with self.assertRaisesRegex(ValueError, "SHM handle signature"):
|
||||
reader.get(address + 1, monotonic_id, signature, "key")
|
||||
|
||||
def test_writer_rejects_handle_no_longer_in_key_index(self):
|
||||
address, monotonic_id = self.storage.put("key", "value")
|
||||
signature = self.storage.get_signature("key")
|
||||
self.storage.verify_signature("key", address, monotonic_id, signature)
|
||||
|
||||
self.storage.clear()
|
||||
|
||||
with self.assertRaisesRegex(ValueError, "SHM handle signature"):
|
||||
self.storage.verify_signature("key", address, monotonic_id, signature)
|
||||
|
||||
def test_handle_issued_before_clear_reads_own_data_until_reused(self):
|
||||
address, monotonic_id = self.storage.put("key", "value")
|
||||
signature = self.storage.get_signature("key")
|
||||
reader = self._open_reader()
|
||||
|
||||
self.storage.clear()
|
||||
|
||||
# Requests still draining after a clear keep reading their own data.
|
||||
self.assertEqual(reader.get(address, monotonic_id, signature, "key"), "value")
|
||||
|
||||
self.storage.put("key", "new_value")
|
||||
|
||||
with self.assertRaisesRegex(ValueError, "modified or is invalid"):
|
||||
reader.get(address, monotonic_id, signature, "key")
|
||||
with self.assertRaisesRegex(ValueError, "modified or is invalid"):
|
||||
reader.touch("key", address, monotonic_id, signature)
|
||||
|
||||
|
||||
# Reader process function
|
||||
@@ -242,11 +300,11 @@ def reader_process(process_id, storage_handle, items_to_read):
|
||||
|
||||
errors = []
|
||||
|
||||
for key, original_value, address, monotonic_id in items_to_read:
|
||||
for key, original_value, address, monotonic_id, signature in items_to_read:
|
||||
time.sleep(random.random() / 100)
|
||||
try:
|
||||
# Read data from shared memory
|
||||
retrieved_value = reader_storage.get(address, monotonic_id)
|
||||
retrieved_value = reader_storage.get(address, monotonic_id, signature, key)
|
||||
|
||||
# Verify data integrity
|
||||
assert retrieved_value == original_value
|
||||
@@ -289,7 +347,8 @@ def run_multiprocess_example():
|
||||
for key, value in test_data:
|
||||
print(f"Storing {key}: {value}")
|
||||
address, monotonic_id = storage.put(key, value)
|
||||
stored_items.append((key, value, address, monotonic_id))
|
||||
signature = storage.get_signature(key)
|
||||
stored_items.append((key, value, address, monotonic_id, signature))
|
||||
print(f" -> Stored at address {address}, ID {monotonic_id}")
|
||||
|
||||
print("\n--- Retrieving Data ---")
|
||||
|
||||
@@ -15,14 +15,25 @@ from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
||||
from vllm.entrypoints.scale_out.token_in_token_out.mm_features import (
|
||||
placeholder_ranges_from_engine_input,
|
||||
)
|
||||
from vllm.entrypoints.scale_out.token_in_token_out.mm_serde import (
|
||||
encode_mm_kwargs_item,
|
||||
)
|
||||
from vllm.entrypoints.scale_out.token_in_token_out.protocol import (
|
||||
GenerateRequest,
|
||||
GenerateResponse,
|
||||
MultiModalFeatures,
|
||||
PlaceholderRangeInfo,
|
||||
)
|
||||
from vllm.entrypoints.scale_out.token_in_token_out.serving import ServingTokens
|
||||
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
|
||||
from vllm.exceptions import GenerationError
|
||||
from vllm.logprobs import Logprob
|
||||
from vllm.multimodal.inputs import PlaceholderRange
|
||||
from vllm.multimodal.inputs import (
|
||||
MultiModalBatchedField,
|
||||
MultiModalFieldElem,
|
||||
MultiModalKwargsItem,
|
||||
PlaceholderRange,
|
||||
)
|
||||
from vllm.outputs import CompletionOutput, RequestOutput
|
||||
from vllm.renderers import renderer_from_config
|
||||
from vllm.renderers.online_renderer import OnlineRenderer
|
||||
@@ -285,6 +296,46 @@ async def test_non_stream_error():
|
||||
await serving.serve_tokens(request)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_serve_tokens_rejects_invalid_shm_handle_before_engine_handoff():
|
||||
engine = _mock_engine()
|
||||
engine.generate = MagicMock()
|
||||
serving = _build_serving_tokens(engine)
|
||||
cache = MagicMock()
|
||||
cache.validate_input_item.side_effect = ValueError(
|
||||
"Invalid SHM handle signature for cache key."
|
||||
)
|
||||
serving.online_renderer.renderer._mm_processor_cache = cache
|
||||
forged_item = MultiModalKwargsItem(
|
||||
{
|
||||
"address": MultiModalFieldElem(data=224, field=MultiModalBatchedField()),
|
||||
"monotonic_id": MultiModalFieldElem(
|
||||
data=4242,
|
||||
field=MultiModalBatchedField(),
|
||||
),
|
||||
}
|
||||
)
|
||||
request = GenerateRequest(
|
||||
token_ids=[1, 2, 3],
|
||||
sampling_params=SamplingParams(max_tokens=1),
|
||||
model=MODEL_NAME,
|
||||
features=MultiModalFeatures(
|
||||
mm_hashes={"image": ["image_A"]},
|
||||
mm_placeholders={"image": [PlaceholderRangeInfo(offset=0, length=1)]},
|
||||
kwargs_data={"image": [encode_mm_kwargs_item(forged_item)]},
|
||||
),
|
||||
)
|
||||
|
||||
response = await serving.serve_tokens(request)
|
||||
|
||||
assert isinstance(response, ErrorResponse)
|
||||
cache.validate_input_item.assert_called_once()
|
||||
validated_item, validated_hash = cache.validate_input_item.call_args.args
|
||||
assert validated_hash == "image_A"
|
||||
assert validated_item["address"].data == 224
|
||||
engine.generate.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stream_basic():
|
||||
"""Streaming returns SSE chunks with correct token_ids and ends with [DONE]."""
|
||||
|
||||
@@ -29,6 +29,7 @@ from vllm.multimodal.cache.base import (
|
||||
)
|
||||
from vllm.multimodal.hasher import MultiModalHasher
|
||||
from vllm.multimodal.inputs import (
|
||||
MultiModalBatchedField,
|
||||
MultiModalFeatureSpec,
|
||||
MultiModalFieldElem,
|
||||
MultiModalKwargsItem,
|
||||
@@ -42,6 +43,7 @@ from vllm.utils.async_utils import make_async
|
||||
from vllm.utils.mem_constants import GiB_bytes, MiB_bytes
|
||||
from vllm.v1.engine.async_llm import AsyncLLM
|
||||
from vllm.v1.engine.llm_engine import LLMEngine
|
||||
from vllm.v1.serial_utils import MsgpackDecoder, MsgpackEncoder
|
||||
|
||||
from ..models.utils import build_model_context
|
||||
from ..utils import create_new_process_for_each_test
|
||||
@@ -778,6 +780,212 @@ def test_cache_eviction_shm_cache():
|
||||
_run_test_cache_eviction_shm(sender_cache, receiver_cache, base_item_size=MiB_bytes)
|
||||
|
||||
|
||||
def test_shm_receiver_requires_sender_handle_for_current_cache_key():
|
||||
vllm_config = VllmConfig(
|
||||
model_config=ModelConfig(
|
||||
model="llava-hf/llava-onevision-qwen2-0.5b-ov-hf",
|
||||
mm_processor_cache_type="shm",
|
||||
mm_shm_cache_max_object_size_mb=1,
|
||||
mm_processor_cache_gb=2 * MiB_bytes / GiB_bytes,
|
||||
),
|
||||
)
|
||||
sender_cache = ShmObjectStoreSenderCache(vllm_config)
|
||||
receiver_cache = ShmObjectStoreReceiverCache(vllm_config, mp.Lock())
|
||||
mm_hash = "image_A"
|
||||
item = MultiModalKwargsItem.dummy(1024)
|
||||
|
||||
try:
|
||||
sender_item, _ = sender_cache.get_and_update_item((item, []), mm_hash)
|
||||
|
||||
valid_feature = MultiModalFeatureSpec(
|
||||
data=sender_item,
|
||||
modality="image",
|
||||
identifier=mm_hash,
|
||||
mm_position=PlaceholderRange(offset=0, length=1),
|
||||
mm_hash=mm_hash,
|
||||
)
|
||||
receiver_cache.get_and_update_features([valid_feature])
|
||||
assert valid_feature.data == item
|
||||
|
||||
forged_item = MultiModalKwargsItem(
|
||||
{
|
||||
"address": sender_item["address"],
|
||||
"monotonic_id": sender_item["monotonic_id"],
|
||||
}
|
||||
)
|
||||
forged_feature = MultiModalFeatureSpec(
|
||||
data=forged_item,
|
||||
modality="image",
|
||||
identifier=mm_hash,
|
||||
mm_position=PlaceholderRange(offset=0, length=1),
|
||||
mm_hash=mm_hash,
|
||||
)
|
||||
with pytest.raises(ValueError, match="SHM handle signature"):
|
||||
receiver_cache.get_and_update_features([forged_feature])
|
||||
|
||||
wrong_key = "image_B"
|
||||
wrong_key_feature = MultiModalFeatureSpec(
|
||||
data=sender_item,
|
||||
modality="image",
|
||||
identifier=wrong_key,
|
||||
mm_position=PlaceholderRange(offset=0, length=1),
|
||||
mm_hash=wrong_key,
|
||||
)
|
||||
with pytest.raises(ValueError, match="SHM handle signature"):
|
||||
receiver_cache.get_and_update_features([wrong_key_feature])
|
||||
finally:
|
||||
receiver_cache._shm_cache.close()
|
||||
sender_cache.close()
|
||||
|
||||
|
||||
def test_shm_sender_handle_survives_engine_msgpack_round_trip():
|
||||
vllm_config = VllmConfig(
|
||||
model_config=ModelConfig(
|
||||
model="llava-hf/llava-onevision-qwen2-0.5b-ov-hf",
|
||||
mm_processor_cache_type="shm",
|
||||
mm_shm_cache_max_object_size_mb=1,
|
||||
mm_processor_cache_gb=2 * MiB_bytes / GiB_bytes,
|
||||
),
|
||||
)
|
||||
sender_cache = ShmObjectStoreSenderCache(vllm_config)
|
||||
receiver_cache = ShmObjectStoreReceiverCache(vllm_config, mp.Lock())
|
||||
mm_hash = "image_A"
|
||||
item = MultiModalKwargsItem.dummy(1024)
|
||||
|
||||
try:
|
||||
sender_item, _ = sender_cache.get_and_update_item((item, []), mm_hash)
|
||||
encoded = MsgpackEncoder().encode(sender_item)
|
||||
decoded = MsgpackDecoder(MultiModalKwargsItem).decode(encoded)
|
||||
feature = MultiModalFeatureSpec(
|
||||
data=decoded,
|
||||
modality="image",
|
||||
identifier=mm_hash,
|
||||
mm_position=PlaceholderRange(offset=0, length=1),
|
||||
mm_hash=mm_hash,
|
||||
)
|
||||
|
||||
receiver_cache.get_and_update_features([feature])
|
||||
|
||||
assert feature.data == item
|
||||
finally:
|
||||
receiver_cache._shm_cache.close()
|
||||
sender_cache.close()
|
||||
|
||||
|
||||
def _small_shm_vllm_config() -> VllmConfig:
|
||||
return VllmConfig(
|
||||
model_config=ModelConfig(
|
||||
model="llava-hf/llava-onevision-qwen2-0.5b-ov-hf",
|
||||
mm_processor_cache_type="shm",
|
||||
mm_shm_cache_max_object_size_mb=1,
|
||||
mm_processor_cache_gb=2 * MiB_bytes / GiB_bytes,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _with_elem(item: MultiModalKwargsItem, key: str, data) -> MultiModalKwargsItem:
|
||||
elems = dict(item)
|
||||
elems[key] = MultiModalFieldElem(data=data, field=MultiModalBatchedField())
|
||||
return MultiModalKwargsItem(elems)
|
||||
|
||||
|
||||
def test_shm_sender_validates_client_supplied_handles():
|
||||
sender_cache = ShmObjectStoreSenderCache(_small_shm_vllm_config())
|
||||
mm_hash = "image_A"
|
||||
|
||||
try:
|
||||
handle, _ = sender_cache.get_and_update_item(
|
||||
(MultiModalKwargsItem.dummy(1024), []), mm_hash
|
||||
)
|
||||
sender_cache.validate_input_item(handle, mm_hash)
|
||||
sender_cache.validate_input_item(MultiModalKwargsItem.dummy(16), mm_hash)
|
||||
|
||||
signature = handle["signature"].data
|
||||
forged = [
|
||||
_with_elem(handle, "signature", [signature[0] ^ 1, *signature[1:]]),
|
||||
_with_elem(handle, "signature", [signature]),
|
||||
_with_elem(handle, "address", True),
|
||||
_with_elem(handle, "address", -1),
|
||||
_with_elem(handle, "monotonic_id", 2**32),
|
||||
MultiModalKwargsItem(
|
||||
{"address": handle["address"], "monotonic_id": handle["monotonic_id"]}
|
||||
),
|
||||
]
|
||||
for item in forged:
|
||||
with pytest.raises(ValueError, match="SHM handle signature"):
|
||||
sender_cache.validate_input_item(item, mm_hash)
|
||||
with pytest.raises(ValueError, match="SHM handle signature"):
|
||||
sender_cache.validate_input_item(handle, "image_B")
|
||||
|
||||
sender_cache.clear_cache()
|
||||
with pytest.raises(ValueError, match="SHM handle signature"):
|
||||
sender_cache.validate_input_item(handle, mm_hash)
|
||||
finally:
|
||||
sender_cache.close()
|
||||
|
||||
|
||||
def test_shm_receiver_rejects_out_of_range_handle_fields():
|
||||
vllm_config = _small_shm_vllm_config()
|
||||
sender_cache = ShmObjectStoreSenderCache(vllm_config)
|
||||
receiver_cache = ShmObjectStoreReceiverCache(vllm_config, mp.Lock())
|
||||
mm_hash = "image_A"
|
||||
|
||||
try:
|
||||
handle, _ = sender_cache.get_and_update_item(
|
||||
(MultiModalKwargsItem.dummy(1024), []), mm_hash
|
||||
)
|
||||
for key, value in [
|
||||
("address", -1),
|
||||
("address", 2**64),
|
||||
("monotonic_id", -1),
|
||||
("monotonic_id", 2**32),
|
||||
]:
|
||||
with pytest.raises(ValueError, match="SHM handle signature"):
|
||||
receiver_cache.get_and_update_item(
|
||||
_with_elem(handle, key, value), mm_hash
|
||||
)
|
||||
finally:
|
||||
receiver_cache._shm_cache.close()
|
||||
sender_cache.close()
|
||||
|
||||
|
||||
def test_shm_handle_issued_before_cache_clear():
|
||||
vllm_config = _small_shm_vllm_config()
|
||||
sender_cache = ShmObjectStoreSenderCache(vllm_config)
|
||||
receiver_cache = ShmObjectStoreReceiverCache(vllm_config, mp.Lock())
|
||||
mm_hash = "image_A"
|
||||
item = MultiModalKwargsItem.dummy(1024)
|
||||
|
||||
def feature(data: MultiModalKwargsItem) -> MultiModalFeatureSpec:
|
||||
return MultiModalFeatureSpec(
|
||||
data=data,
|
||||
modality="image",
|
||||
identifier=mm_hash,
|
||||
mm_position=PlaceholderRange(offset=0, length=1),
|
||||
mm_hash=mm_hash,
|
||||
)
|
||||
|
||||
try:
|
||||
stale_item, _ = sender_cache.get_and_update_item((item, []), mm_hash)
|
||||
sender_cache.clear_cache()
|
||||
|
||||
# A request still draining after the clear resolves to its own data.
|
||||
draining = feature(stale_item)
|
||||
receiver_cache.get_and_update_features([draining])
|
||||
assert draining.data == item
|
||||
|
||||
sender_cache.get_and_update_item(
|
||||
(MultiModalKwargsItem.dummy(2048), []),
|
||||
mm_hash,
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match="modified or is invalid"):
|
||||
receiver_cache.get_and_update_features([feature(stale_item)])
|
||||
finally:
|
||||
receiver_cache._shm_cache.close()
|
||||
sender_cache.close()
|
||||
|
||||
|
||||
def test_processor_cache_shared_across_loras():
|
||||
"""Test that processor cache uses mm_hash to share data across LoRAs."""
|
||||
model_config = ModelConfig(
|
||||
|
||||
@@ -1,7 +1,10 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import hashlib
|
||||
import hmac
|
||||
import pickle
|
||||
import secrets
|
||||
from abc import ABC, abstractmethod
|
||||
from collections.abc import Callable, Iterable
|
||||
from contextlib import contextmanager, suppress
|
||||
@@ -97,6 +100,9 @@ class SingleWriterShmRingBuffer:
|
||||
- Reader synchronization handled by is_free_fn callback
|
||||
- Writer handles garbage collection (free_buf) based on reader feedback
|
||||
|
||||
Shared Memory Layout:
|
||||
`[32-byte handle-signing secret][ring buffer chunks...]`
|
||||
|
||||
Memory Layout per Buffer Chunk:
|
||||
`[4-byte monotonic_id][4-byte chunk_size][actual_data...]`
|
||||
^metadata_start ^data_start
|
||||
@@ -119,6 +125,8 @@ class SingleWriterShmRingBuffer:
|
||||
self.SIZE_NBYTES = 4
|
||||
# 4 bytes for id, 4 bytes for buffer size
|
||||
self.MD_SIZE = self.ID_NBYTES + self.SIZE_NBYTES
|
||||
self.AUTH_SECRET_BYTES = 32
|
||||
self.DATA_OFFSET = self.AUTH_SECRET_BYTES
|
||||
self.monotonic_id_end = 0
|
||||
self.monotonic_id_start = 0
|
||||
self.data_buffer_start = 0
|
||||
@@ -129,8 +137,11 @@ class SingleWriterShmRingBuffer:
|
||||
# we are creating a buffer
|
||||
self.metadata: dict[int, int] = {} # monotonic_id -> start address
|
||||
self.shared_memory = shared_memory.SharedMemory(
|
||||
create=True, size=self.data_buffer_size, name=name
|
||||
create=True,
|
||||
size=self.data_buffer_size + self.DATA_OFFSET,
|
||||
name=name,
|
||||
)
|
||||
self._init_auth_secret()
|
||||
else:
|
||||
# we are opening an existing buffer
|
||||
# fix to https://stackoverflow.com/q/62748654/9191338
|
||||
@@ -146,7 +157,9 @@ class SingleWriterShmRingBuffer:
|
||||
# so the shared memory block size may be larger or equal
|
||||
# to the requested size. The size parameter is ignored
|
||||
# when attaching to an existing block.
|
||||
assert self.shared_memory.size >= self.data_buffer_size
|
||||
assert (
|
||||
self.shared_memory.size >= self.data_buffer_size + self.DATA_OFFSET
|
||||
)
|
||||
|
||||
logger.debug(
|
||||
"Shared memory created/opened with name: %s, size: %d",
|
||||
@@ -164,11 +177,25 @@ class SingleWriterShmRingBuffer:
|
||||
"""Clear the ring buffer."""
|
||||
assert self.is_writer, "Only the writer can clear the buffer."
|
||||
self.metadata.clear()
|
||||
self.monotonic_id_end = 0
|
||||
self.monotonic_id_start = 0
|
||||
# Keep IDs increasing so handles issued before the clear (e.g. for
|
||||
# requests still draining) fail the ID check once their chunk is
|
||||
# reused, instead of resolving to a different object.
|
||||
self.monotonic_id_start = self.monotonic_id_end
|
||||
self.data_buffer_start = 0
|
||||
self.data_buffer_end = 0
|
||||
|
||||
def get_auth_secret(self) -> bytes:
|
||||
"""Return the current handle-signing secret."""
|
||||
assert self.shared_memory.buf is not None, "Buffer has been closed"
|
||||
return bytes(self.shared_memory.buf[: self.AUTH_SECRET_BYTES])
|
||||
|
||||
def _init_auth_secret(self) -> None:
|
||||
assert self.is_writer, "Only the writer can set the auth secret."
|
||||
assert self.shared_memory.buf is not None, "Buffer has been closed"
|
||||
self.shared_memory.buf[: self.AUTH_SECRET_BYTES] = secrets.token_bytes(
|
||||
self.AUTH_SECRET_BYTES
|
||||
)
|
||||
|
||||
def close(self) -> None:
|
||||
"""Close the shared memory."""
|
||||
if hasattr(self, "shared_memory"):
|
||||
@@ -219,13 +246,14 @@ class SingleWriterShmRingBuffer:
|
||||
|
||||
# first 4 bytes as the monotonic id
|
||||
buf_idx = self.data_buffer_end % self.data_buffer_size
|
||||
self.shared_memory.buf[buf_idx : buf_idx + self.ID_NBYTES] = self.int2byte(
|
||||
self.monotonic_id_end
|
||||
physical_idx = self.DATA_OFFSET + buf_idx
|
||||
self.shared_memory.buf[physical_idx : physical_idx + self.ID_NBYTES] = (
|
||||
self.int2byte(self.monotonic_id_end)
|
||||
)
|
||||
# next 4 bytes as the size of the data buffer
|
||||
self.shared_memory.buf[buf_idx + self.ID_NBYTES : buf_idx + self.MD_SIZE] = (
|
||||
self.int2byte(size)
|
||||
)
|
||||
self.shared_memory.buf[
|
||||
physical_idx + self.ID_NBYTES : physical_idx + self.MD_SIZE
|
||||
] = self.int2byte(size)
|
||||
|
||||
# record metadata
|
||||
self.metadata[self.monotonic_id_end % self.ID_MAX] = self.data_buffer_end
|
||||
@@ -240,14 +268,19 @@ class SingleWriterShmRingBuffer:
|
||||
def access_buf(self, address: int):
|
||||
assert self.shared_memory.buf is not None, "Buffer has been closed"
|
||||
buf_idx = address % self.data_buffer_size
|
||||
physical_idx = self.DATA_OFFSET + buf_idx
|
||||
|
||||
# read metadata
|
||||
metadata_buff = self.shared_memory.buf[buf_idx : buf_idx + self.MD_SIZE]
|
||||
metadata_buff = self.shared_memory.buf[
|
||||
physical_idx : physical_idx + self.MD_SIZE
|
||||
]
|
||||
id = self.byte2int(metadata_buff[: self.ID_NBYTES])
|
||||
size = self.byte2int(metadata_buff[self.ID_NBYTES : self.MD_SIZE])
|
||||
|
||||
# yield the data buffer and metadata
|
||||
data_buff = self.shared_memory.buf[buf_idx + self.MD_SIZE : buf_idx + size]
|
||||
data_buff = self.shared_memory.buf[
|
||||
physical_idx + self.MD_SIZE : physical_idx + size
|
||||
]
|
||||
with (
|
||||
memoryview(data_buff) as data_view,
|
||||
):
|
||||
@@ -423,7 +456,8 @@ class SingleWriterShmObjectStorage:
|
||||
|
||||
Architecture:
|
||||
- Single writer process can put(key, value) objects
|
||||
- Multiple reader processes can get(address, monotonic_id) objects
|
||||
- Multiple reader processes can get(address, monotonic_id, signature, key)
|
||||
objects
|
||||
- Built on SingleWriterShmRingBuffer for efficient shared memory management
|
||||
- Thread-safe operations with reader synchronization via locks
|
||||
|
||||
@@ -439,6 +473,13 @@ class SingleWriterShmObjectStorage:
|
||||
- Automatic Cleanup: Garbage collection happens transparently during
|
||||
allocation
|
||||
|
||||
Handle Signatures:
|
||||
- Handles are signed with an HMAC of (key, address, monotonic_id), keyed by
|
||||
a random secret stored at the start of the shared memory
|
||||
- The writer signs the handles it issues and rejects stale or forged
|
||||
handles supplied by requests
|
||||
- Readers verify the signature in get/touch before reading shared memory
|
||||
|
||||
Memory Layout per Object:
|
||||
`[4-byte reference_count][metadata_size][serialized_object_data]`
|
||||
|
||||
@@ -521,6 +562,76 @@ class SingleWriterShmObjectStorage:
|
||||
else:
|
||||
raise ValueError(f"Unsupported data type for serialization: {type(data)}")
|
||||
|
||||
def _validate_monotonic_id(
|
||||
self,
|
||||
address: int,
|
||||
monotonic_id: int,
|
||||
buf_metadata: tuple[int, int],
|
||||
) -> None:
|
||||
if buf_metadata[0] != monotonic_id:
|
||||
raise ValueError(
|
||||
f"Data for address:id '{address}:{monotonic_id}'"
|
||||
" has been modified or is invalid."
|
||||
)
|
||||
|
||||
def _verify_reader_signature(
|
||||
self,
|
||||
address: int,
|
||||
monotonic_id: int,
|
||||
signature: list[int] | None,
|
||||
key: str | None,
|
||||
) -> None:
|
||||
if signature is None or key is None:
|
||||
raise ValueError("Missing SHM handle signature for cache key.")
|
||||
if (
|
||||
not isinstance(signature, list)
|
||||
or len(signature) != hashlib.sha256().digest_size
|
||||
or any(type(byte) is not int or not 0 <= byte <= 255 for byte in signature)
|
||||
or not isinstance(key, str)
|
||||
):
|
||||
raise ValueError("Invalid SHM handle signature for cache key.")
|
||||
|
||||
expected_signature = self._make_signature(key, address, monotonic_id)
|
||||
if not hmac.compare_digest(bytes(signature), expected_signature):
|
||||
raise ValueError("Invalid SHM handle signature for cache key.")
|
||||
|
||||
def verify_signature(
|
||||
self,
|
||||
key: str | None,
|
||||
address: int,
|
||||
monotonic_id: int,
|
||||
signature: list[int] | None,
|
||||
) -> None:
|
||||
"""Verify that a handle was issued for the given key.
|
||||
|
||||
For writers: the handle must also be the key's current entry
|
||||
For readers: only the signature is checked
|
||||
|
||||
Args:
|
||||
key: String key the handle belongs to
|
||||
address: Address of the object
|
||||
monotonic_id: Monotonic ID of the object
|
||||
signature: Signature issued with the handle
|
||||
|
||||
"""
|
||||
if self.is_writer and (
|
||||
key is None or self.key_index.get(key) != (address, monotonic_id)
|
||||
):
|
||||
raise ValueError("Invalid SHM handle signature for cache key.")
|
||||
self._verify_reader_signature(address, monotonic_id, signature, key)
|
||||
|
||||
def _make_signature(self, key: str, address: int, monotonic_id: int) -> bytes:
|
||||
payload = (
|
||||
hashlib.sha256(key.encode("utf-8")).digest()
|
||||
+ address.to_bytes(8, "little", signed=False)
|
||||
+ monotonic_id.to_bytes(4, "little", signed=False)
|
||||
)
|
||||
return hmac.new(
|
||||
self.ring_buffer.get_auth_secret(),
|
||||
payload,
|
||||
hashlib.sha256,
|
||||
).digest()
|
||||
|
||||
def increment_writer_flag(self, id: int) -> None:
|
||||
"""Set the in-use flag for the writer."""
|
||||
self.writer_flag[id] = self.writer_flag.get(id, 0) + 1
|
||||
@@ -555,6 +666,11 @@ class SingleWriterShmObjectStorage:
|
||||
self.increment_writer_flag(monotonic_id)
|
||||
return address, monotonic_id
|
||||
|
||||
def get_signature(self, key: str) -> list[int]:
|
||||
"""Sign the handle of a cached object so readers can verify it."""
|
||||
address, monotonic_id = self.key_index[key]
|
||||
return list(self._make_signature(key, address, monotonic_id))
|
||||
|
||||
def put(self, key: str, value: Any) -> tuple[int, int]:
|
||||
"""Store a key-value pair in the object storage.
|
||||
Attempts to free max_object_size bytes using FIFO order
|
||||
@@ -605,15 +721,20 @@ class SingleWriterShmObjectStorage:
|
||||
self.id_index[monotonic_id] = key
|
||||
return address, monotonic_id
|
||||
|
||||
def get(self, address: int, monotonic_id: int) -> Any:
|
||||
def get(
|
||||
self,
|
||||
address: int,
|
||||
monotonic_id: int,
|
||||
signature: list[int] | None = None,
|
||||
key: str | None = None,
|
||||
) -> Any:
|
||||
if not self.is_writer:
|
||||
# Reject forged handles before dereferencing the supplied address.
|
||||
self.verify_signature(key, address, monotonic_id, signature)
|
||||
|
||||
# Read data from buffer
|
||||
with self.ring_buffer.access_buf(address) as (data_view, buf_metadata):
|
||||
# check id from metadata
|
||||
if buf_metadata[0] != monotonic_id:
|
||||
raise ValueError(
|
||||
f"Data for address:id '{address}:{monotonic_id}'"
|
||||
" has been modified or is invalid."
|
||||
)
|
||||
self._validate_monotonic_id(address, monotonic_id, buf_metadata)
|
||||
|
||||
obj = self.ser_de.deserialize(data_view[self.flag_bytes :])
|
||||
|
||||
@@ -633,6 +754,7 @@ class SingleWriterShmObjectStorage:
|
||||
key: str,
|
||||
address: int = 0,
|
||||
monotonic_id: int = 0,
|
||||
signature: list[int] | None = None,
|
||||
) -> None:
|
||||
"""Touch an existing cached item to update its eviction status.
|
||||
|
||||
@@ -643,6 +765,7 @@ class SingleWriterShmObjectStorage:
|
||||
key: String key of the object to touch
|
||||
address: Address of the object (only for readers)
|
||||
monotonic_id: Monotonic ID of the object (only for readers)
|
||||
signature: Server-issued handle signature (only for readers)
|
||||
|
||||
"""
|
||||
if self._reader_lock is None:
|
||||
@@ -652,10 +775,13 @@ class SingleWriterShmObjectStorage:
|
||||
# Writer side: increment writer_flag to raise eviction threshold
|
||||
self.increment_writer_flag(monotonic_id)
|
||||
else:
|
||||
# Reject forged handles before dereferencing the supplied address.
|
||||
self.verify_signature(key, address, monotonic_id, signature)
|
||||
with (
|
||||
self._reader_lock,
|
||||
self.ring_buffer.access_buf(address) as (data_view, _),
|
||||
self.ring_buffer.access_buf(address) as (data_view, buf_metadata),
|
||||
):
|
||||
self._validate_monotonic_id(address, monotonic_id, buf_metadata)
|
||||
reader_count = self.ring_buffer.byte2int(data_view[: self.flag_bytes])
|
||||
|
||||
# NOTE(Long):
|
||||
|
||||
@@ -39,6 +39,7 @@ from vllm.inputs import EngineInput, TokensPrompt, mm_input
|
||||
from vllm.logger import init_logger
|
||||
from vllm.logprobs import Logprob
|
||||
from vllm.multimodal.inputs import (
|
||||
MultiModalKwargsItem,
|
||||
MultiModalKwargsItems,
|
||||
PlaceholderRange,
|
||||
)
|
||||
@@ -106,6 +107,23 @@ class ServingTokens(GenerateBaseServing):
|
||||
else getattr(mc, "override_generation_config", {}).get("max_new_tokens")
|
||||
)
|
||||
|
||||
def _validate_mm_cache_handles(
|
||||
self,
|
||||
mm_kwargs: dict[str, list[MultiModalKwargsItem | None]],
|
||||
mm_hashes: dict[str, list[str]],
|
||||
) -> ErrorResponse | None:
|
||||
cache = self.online_renderer.renderer.mm_processor_cache
|
||||
if cache is None:
|
||||
return None
|
||||
try:
|
||||
for modality, items in mm_kwargs.items():
|
||||
for mm_hash, item in zip(mm_hashes[modality], items, strict=True):
|
||||
if item is not None:
|
||||
cache.validate_input_item(item, mm_hash)
|
||||
except ValueError as error:
|
||||
return self.create_error_response(error)
|
||||
return None
|
||||
|
||||
async def serve_tokens(
|
||||
self,
|
||||
request: GenerateRequest,
|
||||
@@ -201,6 +219,8 @@ class ServingTokens(GenerateBaseServing):
|
||||
# Deserialize full tensor data and optional metadata-only data.
|
||||
# Metadata-only items are valid when ec_transfer_params is set.
|
||||
mm_kwargs = mm_kwargs_from_features(features)
|
||||
if error := self._validate_mm_cache_handles(mm_kwargs, features.mm_hashes):
|
||||
return error
|
||||
|
||||
engine_input = mm_input(
|
||||
prompt_token_ids=request.token_ids,
|
||||
|
||||
Vendored
+8
@@ -356,6 +356,14 @@ class BaseMultiModalProcessorCache(
|
||||
"""Close the underlying cache, if needed."""
|
||||
pass
|
||||
|
||||
def validate_input_item(
|
||||
self,
|
||||
mm_item: MultiModalKwargsItem,
|
||||
mm_hash: str,
|
||||
) -> None:
|
||||
"""Validate externally supplied cache metadata before engine handoff."""
|
||||
return None
|
||||
|
||||
@abstractmethod
|
||||
def touch_sender_cache_item(self, mm_hash: str) -> None:
|
||||
"""Update the cache eviction order for a multi-modal item.
|
||||
|
||||
Vendored
+73
-12
@@ -4,7 +4,7 @@
|
||||
|
||||
from collections.abc import Sequence
|
||||
from multiprocessing.synchronize import Lock as LockType
|
||||
from typing import TYPE_CHECKING, cast
|
||||
from typing import TYPE_CHECKING, TypeAlias
|
||||
|
||||
from typing_extensions import override
|
||||
|
||||
@@ -37,6 +37,29 @@ if TYPE_CHECKING:
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
ShmHandle: TypeAlias = tuple[int, int, list[int]]
|
||||
|
||||
|
||||
def _get_shm_handle(mm_item: MultiModalKwargsItem) -> ShmHandle | None:
|
||||
if "address" not in mm_item:
|
||||
return None
|
||||
if "monotonic_id" not in mm_item or "signature" not in mm_item:
|
||||
raise ValueError("Missing SHM handle signature for cache key.")
|
||||
|
||||
address = mm_item["address"].data
|
||||
monotonic_id = mm_item["monotonic_id"].data
|
||||
signature = mm_item["signature"].data
|
||||
if (
|
||||
type(address) is not int
|
||||
or not 0 <= address < 2**64
|
||||
or type(monotonic_id) is not int
|
||||
or not 0 <= monotonic_id < 2**32
|
||||
or not isinstance(signature, list)
|
||||
):
|
||||
raise ValueError("Invalid SHM handle signature for cache key.")
|
||||
|
||||
return address, monotonic_id, signature
|
||||
|
||||
|
||||
class ShmObjectStoreSenderCache(BaseMultiModalProcessorCache):
|
||||
"""The cache which is used on P0 when SHM caching is enabled.
|
||||
@@ -98,8 +121,12 @@ class ShmObjectStoreSenderCache(BaseMultiModalProcessorCache):
|
||||
self._total += 1
|
||||
|
||||
address, monotonic_id = self._shm_cache.get_cached(mm_hash)
|
||||
signature = self._shm_cache.get_signature(mm_hash)
|
||||
prompt_updates = self._p0_cache[mm_hash]
|
||||
return self.address_as_item(address, monotonic_id), prompt_updates
|
||||
return (
|
||||
self.address_as_item(address, monotonic_id, signature),
|
||||
prompt_updates,
|
||||
)
|
||||
|
||||
assert mm_item is not None, f"Expected a cached item for {mm_hash=}"
|
||||
item, prompt_updates = mm_item
|
||||
@@ -108,12 +135,16 @@ class ShmObjectStoreSenderCache(BaseMultiModalProcessorCache):
|
||||
|
||||
try:
|
||||
address, monotonic_id = self._shm_cache.put(mm_hash, item)
|
||||
signature = self._shm_cache.get_signature(mm_hash)
|
||||
# Try to remove dangling items if p0 cache is too large.
|
||||
if len(self._p0_cache) >= 2 * len(self._shm_cache.key_index):
|
||||
self.remove_dangling_items()
|
||||
|
||||
self._p0_cache[mm_hash] = prompt_updates
|
||||
return self.address_as_item(address, monotonic_id), prompt_updates
|
||||
return (
|
||||
self.address_as_item(address, monotonic_id, signature),
|
||||
prompt_updates,
|
||||
)
|
||||
except ValueError as e:
|
||||
# `put` raises ValueError either for an oversize item or for a
|
||||
# duplicate key (concurrent insert); the latter is benign so we
|
||||
@@ -143,6 +174,21 @@ class ShmObjectStoreSenderCache(BaseMultiModalProcessorCache):
|
||||
Increments writer_flag on sender side."""
|
||||
self._shm_cache.touch(mm_hash)
|
||||
|
||||
@override
|
||||
def validate_input_item(
|
||||
self,
|
||||
mm_item: MultiModalKwargsItem,
|
||||
mm_hash: str,
|
||||
) -> None:
|
||||
if (handle := _get_shm_handle(mm_item)) is not None:
|
||||
address, monotonic_id, signature = handle
|
||||
self._shm_cache.verify_signature(
|
||||
mm_hash,
|
||||
address,
|
||||
monotonic_id,
|
||||
signature,
|
||||
)
|
||||
|
||||
@override
|
||||
def clear_cache(self) -> None:
|
||||
self._shm_cache.clear()
|
||||
@@ -171,6 +217,7 @@ class ShmObjectStoreSenderCache(BaseMultiModalProcessorCache):
|
||||
self,
|
||||
address: int,
|
||||
monotonic_id: int,
|
||||
signature: list[int],
|
||||
) -> MultiModalKwargsItem:
|
||||
addr_elem = MultiModalFieldElem(
|
||||
data=address,
|
||||
@@ -181,7 +228,18 @@ class ShmObjectStoreSenderCache(BaseMultiModalProcessorCache):
|
||||
field=MultiModalBatchedField(),
|
||||
)
|
||||
|
||||
return MultiModalKwargsItem({"address": addr_elem, "monotonic_id": id_elem})
|
||||
signature_elem = MultiModalFieldElem(
|
||||
data=signature,
|
||||
field=MultiModalBatchedField(),
|
||||
)
|
||||
|
||||
return MultiModalKwargsItem(
|
||||
{
|
||||
"address": addr_elem,
|
||||
"monotonic_id": id_elem,
|
||||
"signature": signature_elem,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
class ShmObjectStoreReceiverCache(BaseMultiModalReceiverCache):
|
||||
@@ -235,10 +293,9 @@ class ShmObjectStoreReceiverCache(BaseMultiModalReceiverCache):
|
||||
mm_hash: str,
|
||||
) -> MultiModalKwargsItem:
|
||||
assert mm_item is not None, f"Expected an address item for {mm_hash=}"
|
||||
if "address" in mm_item:
|
||||
address = cast(int, mm_item["address"].data)
|
||||
monotonic_id = cast(int, mm_item["monotonic_id"].data)
|
||||
return self._shm_cache.get(address, monotonic_id)
|
||||
if (handle := _get_shm_handle(mm_item)) is not None:
|
||||
address, monotonic_id, signature = handle
|
||||
return self._shm_cache.get(address, monotonic_id, signature, mm_hash)
|
||||
|
||||
return mm_item
|
||||
|
||||
@@ -251,10 +308,14 @@ class ShmObjectStoreReceiverCache(BaseMultiModalReceiverCache):
|
||||
"""Touch the item in shared memory cache to prevent eviction.
|
||||
Increments reader_count on receiver side."""
|
||||
assert mm_item is not None
|
||||
if "address" in mm_item:
|
||||
address = cast(int, mm_item["address"].data)
|
||||
monotonic_id = cast(int, mm_item["monotonic_id"].data)
|
||||
self._shm_cache.touch(mm_hash, address=address, monotonic_id=monotonic_id)
|
||||
if (handle := _get_shm_handle(mm_item)) is not None:
|
||||
address, monotonic_id, signature = handle
|
||||
self._shm_cache.touch(
|
||||
mm_hash,
|
||||
address=address,
|
||||
monotonic_id=monotonic_id,
|
||||
signature=signature,
|
||||
)
|
||||
|
||||
@override
|
||||
def clear_cache(self) -> None:
|
||||
|
||||
Reference in New Issue
Block a user