[Core] Prevent side-channel attacks via cache salting (#17045)
Signed-off-by: Marko Rosenmueller <5467316+dr75@users.noreply.github.com>
This commit is contained in:
@@ -272,3 +272,43 @@ def test_serving_chat_could_load_correct_generation_config():
|
||||
|
||||
assert mock_engine.generate.call_args.args[1].temperature == 0.0
|
||||
assert mock_engine.generate.call_args.args[1].repetition_penalty == 1.05
|
||||
|
||||
|
||||
def test_serving_chat_did_set_correct_cache_salt():
|
||||
mock_model_config = MockModelConfig()
|
||||
|
||||
mock_engine = MagicMock(spec=MQLLMEngineClient)
|
||||
mock_engine.get_tokenizer.return_value = get_tokenizer(MODEL_NAME)
|
||||
mock_engine.errored = False
|
||||
|
||||
# Initialize the serving chat
|
||||
models = OpenAIServingModels(engine_client=mock_engine,
|
||||
base_model_paths=BASE_MODEL_PATHS,
|
||||
model_config=mock_model_config)
|
||||
serving_chat = OpenAIServingChat(mock_engine,
|
||||
mock_model_config,
|
||||
models,
|
||||
response_role="assistant",
|
||||
chat_template=CHAT_TEMPLATE,
|
||||
chat_template_content_format="auto",
|
||||
request_logger=None)
|
||||
|
||||
# Test cache_salt
|
||||
req = ChatCompletionRequest(
|
||||
model=MODEL_NAME,
|
||||
messages=[{
|
||||
"role": "user",
|
||||
"content": "what is 1+1?"
|
||||
}],
|
||||
)
|
||||
|
||||
# By default cache_salt in the engine prompt is not set
|
||||
with suppress(Exception):
|
||||
asyncio.run(serving_chat.create_chat_completion(req))
|
||||
assert "cache_salt" not in mock_engine.generate.call_args.args[0]
|
||||
|
||||
# Test with certain cache_salt
|
||||
req.cache_salt = "test_salt"
|
||||
with suppress(Exception):
|
||||
asyncio.run(serving_chat.create_chat_completion(req))
|
||||
assert mock_engine.generate.call_args.args[0]["cache_salt"] == "test_salt"
|
||||
|
||||
@@ -60,8 +60,16 @@ def _run_incremental_decode(tokenizer,
|
||||
skip_special_tokens=skip_special_tokens,
|
||||
spaces_between_special_tokens=spaces_between_special_tokens,
|
||||
)
|
||||
request = EngineCoreRequest("", prompt_token_ids, None, None, None, params,
|
||||
None, 0.0, None)
|
||||
request = EngineCoreRequest("",
|
||||
prompt_token_ids,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
params,
|
||||
None,
|
||||
0.0,
|
||||
None,
|
||||
cache_salt=None)
|
||||
|
||||
if fast is None:
|
||||
detokenizer = IncrementalDetokenizer.from_new_request(
|
||||
|
||||
@@ -29,7 +29,8 @@ from vllm.v1.request import Request
|
||||
def make_request(request_id,
|
||||
prompt_token_ids,
|
||||
mm_positions=None,
|
||||
mm_hashes=None):
|
||||
mm_hashes=None,
|
||||
cache_salt=None):
|
||||
if mm_positions is None:
|
||||
multi_modal_inputs = None
|
||||
else:
|
||||
@@ -45,6 +46,7 @@ def make_request(request_id,
|
||||
eos_token_id=100,
|
||||
arrival_time=0,
|
||||
lora_request=None,
|
||||
cache_salt=cache_salt,
|
||||
)
|
||||
|
||||
|
||||
@@ -213,6 +215,45 @@ def test_generate_block_hash_extra_keys_no_mm_inputs():
|
||||
assert next_mm_idx == 0
|
||||
|
||||
|
||||
def test_generate_block_hash_extra_keys_cache_salt():
|
||||
request = make_request(
|
||||
request_id=0,
|
||||
prompt_token_ids=[_ for _ in range(6)],
|
||||
mm_positions=None,
|
||||
mm_hashes=None,
|
||||
cache_salt="salt",
|
||||
)
|
||||
|
||||
# salt is added for the first token
|
||||
extra_keys, _ = generate_block_hash_extra_keys(request, 0, 1, 0)
|
||||
assert extra_keys == ('salt', )
|
||||
extra_keys, _ = generate_block_hash_extra_keys(request, 0, 10, 0)
|
||||
assert extra_keys == ('salt', )
|
||||
|
||||
# no salt added for other tokens
|
||||
extra_keys, _ = generate_block_hash_extra_keys(request, 1, 2, 0)
|
||||
assert extra_keys is None
|
||||
extra_keys, _ = generate_block_hash_extra_keys(request, 6, 10, 0)
|
||||
assert extra_keys is None
|
||||
|
||||
# works together with other extra keys
|
||||
request_mm = make_request(
|
||||
request_id=0,
|
||||
prompt_token_ids=[_ for _ in range(20)],
|
||||
mm_positions=[
|
||||
PlaceholderRange(offset=0, length=5),
|
||||
],
|
||||
mm_hashes=["hash1"],
|
||||
cache_salt="salt",
|
||||
)
|
||||
|
||||
# Test with no extra keys
|
||||
extra_keys, next_mm_idx = generate_block_hash_extra_keys(
|
||||
request_mm, 0, 5, 0)
|
||||
assert extra_keys == ("hash1", "salt")
|
||||
assert next_mm_idx == 1
|
||||
|
||||
|
||||
@pytest.mark.parametrize("hash_fn", [sha256, hash])
|
||||
def test_hash_block_tokens(hash_fn):
|
||||
parent_block_hash = 123
|
||||
|
||||
@@ -21,7 +21,8 @@ def make_request(request_id,
|
||||
prompt_token_ids,
|
||||
mm_positions=None,
|
||||
mm_hashes=None,
|
||||
prompt_logprobs: Optional[int] = None):
|
||||
prompt_logprobs: Optional[int] = None,
|
||||
cache_salt: Optional[str] = None):
|
||||
if mm_positions is None:
|
||||
multi_modal_inputs = None
|
||||
else:
|
||||
@@ -38,6 +39,7 @@ def make_request(request_id,
|
||||
eos_token_id=100,
|
||||
arrival_time=0,
|
||||
lora_request=None,
|
||||
cache_salt=cache_salt,
|
||||
)
|
||||
|
||||
|
||||
@@ -603,6 +605,66 @@ def test_mm_prefix_caching():
|
||||
assert num_computed_tokens == 3 * 16
|
||||
|
||||
|
||||
def test_cache_key_salting():
|
||||
"""
|
||||
This tests that cache salts are applied during hashing and the cache
|
||||
is separated cache as expected.
|
||||
"""
|
||||
block_size = 16
|
||||
manager = KVCacheManager(
|
||||
make_kv_cache_config(block_size, 11),
|
||||
max_model_len=8192,
|
||||
enable_caching=True,
|
||||
)
|
||||
|
||||
# 3 complete blocks and an incomplete block with 11 tokens.
|
||||
common_token_ids = [i for i in range(3) for _ in range(block_size)]
|
||||
token_ids = common_token_ids + [3] * 11
|
||||
req0 = make_request("0", token_ids, cache_salt="salt1")
|
||||
computed_blocks, num_computed_tokens = manager.get_computed_blocks(req0)
|
||||
|
||||
# Completed block should have hashes with extra keys.
|
||||
assert not computed_blocks
|
||||
assert num_computed_tokens == 0
|
||||
block_hashes = manager.req_to_block_hashes[req0.request_id]
|
||||
assert len(block_hashes) == 3
|
||||
assert block_hashes[0].extra_keys == ("salt1", )
|
||||
assert block_hashes[1].extra_keys is None
|
||||
assert block_hashes[2].extra_keys is None
|
||||
|
||||
blocks = manager.allocate_slots(req0, 59, computed_blocks)
|
||||
assert [b.block_id for b in blocks] == [1, 2, 3, 4]
|
||||
req0.num_computed_tokens = 59
|
||||
|
||||
# Append slots without allocating a new block.
|
||||
for _ in range(5):
|
||||
req0.append_output_token_ids(8)
|
||||
new_blocks = manager.allocate_slots(req0, 5)
|
||||
assert new_blocks is not None and len(new_blocks) == 0
|
||||
|
||||
# Now one more block that should not have extra keys.
|
||||
assert len(block_hashes) == 4
|
||||
assert block_hashes[3].extra_keys is None
|
||||
|
||||
# Test cache hit with a new request that has the same salt.
|
||||
token_ids = common_token_ids + [4] * 11
|
||||
req1 = make_request("1", token_ids, cache_salt="salt1")
|
||||
computed_blocks, num_computed_tokens = manager.get_computed_blocks(req1)
|
||||
# Should match only a prefix of 3 blocks.
|
||||
assert len(computed_blocks) == 3
|
||||
assert num_computed_tokens == 3 * block_size
|
||||
|
||||
# Test cache miss with same content but different salt.
|
||||
token_ids = common_token_ids + [4] * 11
|
||||
req2 = make_request("2", token_ids, cache_salt="salt2")
|
||||
computed_blocks, num_computed_tokens = manager.get_computed_blocks(req2)
|
||||
assert len(computed_blocks) == 0
|
||||
assert num_computed_tokens == 0
|
||||
block_hashes = manager.req_to_block_hashes[req2.request_id]
|
||||
assert len(block_hashes) == 3
|
||||
assert block_hashes[0].extra_keys == ("salt2", )
|
||||
|
||||
|
||||
def test_prefill_not_enough_free_blocks_with_computed_blocks():
|
||||
"""
|
||||
This is a unit test that tests the correctness of the allocate_slots
|
||||
|
||||
@@ -40,6 +40,7 @@ def make_request() -> EngineCoreRequest:
|
||||
eos_token_id=None,
|
||||
arrival_time=time.time(),
|
||||
lora_request=None,
|
||||
cache_salt=None,
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -43,6 +43,7 @@ def make_request(params: SamplingParams) -> EngineCoreRequest:
|
||||
eos_token_id=None,
|
||||
arrival_time=time.time(),
|
||||
lora_request=None,
|
||||
cache_salt=None,
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -57,6 +57,7 @@ def test_incremental_detokenization(request_output_kind: RequestOutputKind,
|
||||
mm_placeholders=None,
|
||||
eos_token_id=None,
|
||||
lora_request=None,
|
||||
cache_salt=None,
|
||||
sampling_params=SamplingParams(
|
||||
skip_special_tokens=False,
|
||||
spaces_between_special_tokens=False,
|
||||
@@ -403,6 +404,7 @@ def test_logprobs_processor(request_output_kind: RequestOutputKind,
|
||||
mm_placeholders=None,
|
||||
eos_token_id=None,
|
||||
lora_request=None,
|
||||
cache_salt=None,
|
||||
sampling_params=SamplingParams(
|
||||
skip_special_tokens=False,
|
||||
spaces_between_special_tokens=False,
|
||||
@@ -503,7 +505,7 @@ def test_stop_token(include_stop_str_in_output: bool,
|
||||
reason should be "stop" (i.e. first control token causes stop
|
||||
and is represented in output text)
|
||||
|
||||
* else, the detokenized string should be
|
||||
* else, the detokenized string should be
|
||||
<token><token>...<token> and the finish reason should be "stop"
|
||||
(i.e. first control token causes stop but is not represented
|
||||
in output text.)
|
||||
@@ -565,6 +567,7 @@ def test_stop_token(include_stop_str_in_output: bool,
|
||||
mm_placeholders=None,
|
||||
eos_token_id=eos_token_id,
|
||||
lora_request=None,
|
||||
cache_salt=None,
|
||||
sampling_params=SamplingParams(
|
||||
skip_special_tokens=False,
|
||||
spaces_between_special_tokens=False,
|
||||
@@ -661,6 +664,7 @@ def test_stop_string(include_stop_str_in_output: bool,
|
||||
mm_placeholders=None,
|
||||
eos_token_id=None,
|
||||
lora_request=None,
|
||||
cache_salt=None,
|
||||
sampling_params=SamplingParams(
|
||||
skip_special_tokens=False,
|
||||
spaces_between_special_tokens=False,
|
||||
@@ -774,6 +778,7 @@ def test_iteration_stats(dummy_test_vectors):
|
||||
mm_placeholders=None,
|
||||
eos_token_id=None,
|
||||
lora_request=None,
|
||||
cache_salt=None,
|
||||
sampling_params=SamplingParams(),
|
||||
) for idx, prompt_tokens in enumerate(dummy_test_vectors.prompt_tokens)
|
||||
]
|
||||
|
||||
Reference in New Issue
Block a user