[Core][BugFix] Tag prefix-cache extra keys by source (#51899)
Signed-off-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Lucas Bourtoule <35483370+dhalf@users.noreply.github.com> Co-authored-by: Tai An <antai12232931@outlook.com> Co-authored-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Lucas Bourtoule
Tai An
Nick Hill
Claude Opus 5.5
parent
09c47db1ca
commit
765872e7ed
@@ -54,6 +54,7 @@ from vllm.v1.core.kv_cache_utils import (
|
||||
is_kv_cache_spec_uniform,
|
||||
make_block_hash_with_group_id,
|
||||
tensor_data,
|
||||
to_event_extra_keys,
|
||||
)
|
||||
from vllm.v1.hisparse.layout import (
|
||||
create_hisparse_layout,
|
||||
@@ -822,12 +823,12 @@ def test_generate_block_hash_extra_keys():
|
||||
|
||||
# Test with no extra keys
|
||||
extra_keys, next_mm_idx = generate_block_hash_extra_keys(request, 0, 5, 0)
|
||||
assert extra_keys == (("hash1", 0),)
|
||||
assert extra_keys == (("mm", "hash1", 0),)
|
||||
assert next_mm_idx == 1
|
||||
|
||||
# Test with partial overlap
|
||||
extra_keys, next_mm_idx = generate_block_hash_extra_keys(request, 3, 8, 0)
|
||||
assert extra_keys == (("hash1", -3),)
|
||||
assert extra_keys == (("mm", "hash1", -3),)
|
||||
assert next_mm_idx == 1
|
||||
|
||||
# Test with no overlap
|
||||
@@ -837,7 +838,7 @@ def test_generate_block_hash_extra_keys():
|
||||
|
||||
# Test with multiple extra keys
|
||||
extra_keys, next_mm_idx = generate_block_hash_extra_keys(request, 0, 15, 0)
|
||||
assert extra_keys == (("hash1", 0), ("hash2", 10))
|
||||
assert extra_keys == (("mm", "hash1", 0), ("mm", "hash2", 10))
|
||||
assert next_mm_idx == 2
|
||||
|
||||
|
||||
@@ -865,9 +866,9 @@ def test_generate_block_hash_extra_keys_cache_salt():
|
||||
|
||||
# salt is added for the first token
|
||||
extra_keys, _ = generate_block_hash_extra_keys(request, 0, 1, 0)
|
||||
assert extra_keys == ("salt",)
|
||||
assert extra_keys == (("cache_salt", "salt"),)
|
||||
extra_keys, _ = generate_block_hash_extra_keys(request, 0, 10, 0)
|
||||
assert extra_keys == ("salt",)
|
||||
assert extra_keys == (("cache_salt", "salt"),)
|
||||
|
||||
# no salt added for other tokens
|
||||
extra_keys, _ = generate_block_hash_extra_keys(request, 1, 2, 0)
|
||||
@@ -888,7 +889,7 @@ def test_generate_block_hash_extra_keys_cache_salt():
|
||||
|
||||
# Test with no extra keys
|
||||
extra_keys, next_mm_idx = generate_block_hash_extra_keys(request_mm, 0, 5, 0)
|
||||
assert extra_keys == (("hash1", 0), "salt")
|
||||
assert extra_keys == (("mm", "hash1", 0), ("cache_salt", "salt"))
|
||||
assert next_mm_idx == 1
|
||||
|
||||
|
||||
@@ -906,13 +907,13 @@ def test_generate_block_hash_extra_keys_prompt_embeds():
|
||||
extra_keys, _ = generate_block_hash_extra_keys(request, 0, 5, 0)
|
||||
expected_embeds = prompt_embeds[0:5]
|
||||
expected_hash = hashlib.sha256(kv_cache_utils.tensor_data(expected_embeds)).digest()
|
||||
assert extra_keys == (expected_hash,)
|
||||
assert extra_keys == (("prompt_embeds", expected_hash),)
|
||||
|
||||
# Test with prompt embeds for the second block
|
||||
extra_keys, _ = generate_block_hash_extra_keys(request, 5, 10, 0)
|
||||
expected_embeds = prompt_embeds[5:10]
|
||||
expected_hash = hashlib.sha256(kv_cache_utils.tensor_data(expected_embeds)).digest()
|
||||
assert extra_keys == (expected_hash,)
|
||||
assert extra_keys == (("prompt_embeds", expected_hash),)
|
||||
|
||||
|
||||
def test_generate_block_hash_extra_keys_prompt_embeds_cached(monkeypatch):
|
||||
@@ -976,13 +977,58 @@ def test_generate_block_hash_extra_keys_lora():
|
||||
)
|
||||
|
||||
extra_keys, _ = generate_block_hash_extra_keys(request, 0, 3, 0)
|
||||
assert extra_keys == ("test_lora_adapter",)
|
||||
assert extra_keys == (("lora", "test_lora_adapter"),)
|
||||
|
||||
request.lora_request = None
|
||||
extra_keys, _ = generate_block_hash_extra_keys(request, 0, 3, 0)
|
||||
assert extra_keys is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize("hash_fn", [sha256, sha256_cbor])
|
||||
def test_lora_name_and_cache_salt_block_hashes_do_not_collide(hash_fn):
|
||||
"""A cache_salt equal to a LoRA name must not share that LoRA's blocks."""
|
||||
lora_request = LoRARequest(
|
||||
lora_name="foo", lora_int_id=1, lora_path="/path/to/lora"
|
||||
)
|
||||
lora_req = Request(
|
||||
request_id="lora",
|
||||
prompt_token_ids=[0, 1, 2],
|
||||
sampling_params=SamplingParams(max_tokens=1),
|
||||
pooling_params=None,
|
||||
lora_request=lora_request,
|
||||
block_hasher=get_request_block_hasher(3, hash_fn),
|
||||
)
|
||||
salted_req = make_request("salted", [0, 1, 2], hash_fn=hash_fn, cache_salt="foo")
|
||||
|
||||
assert lora_req.block_hashes[0] != salted_req.block_hashes[0]
|
||||
|
||||
|
||||
def test_to_event_extra_keys_keeps_untagged_event_format():
|
||||
"""KV events keep publishing the extra-key shapes consumers already parse."""
|
||||
request = make_request(
|
||||
"0",
|
||||
list(range(10)),
|
||||
mm_positions=[PlaceholderRange(offset=2, length=3)],
|
||||
mm_hashes=["hash1"],
|
||||
cache_salt="salt",
|
||||
)
|
||||
request.lora_request = LoRARequest(
|
||||
lora_name="adapter", lora_int_id=1, lora_path="/path/to/lora"
|
||||
)
|
||||
|
||||
extra_keys, _ = generate_block_hash_extra_keys(request, 0, 5, 0)
|
||||
|
||||
assert extra_keys == (
|
||||
("lora", "adapter"),
|
||||
("mm", "hash1", 2),
|
||||
("cache_salt", "salt"),
|
||||
)
|
||||
event_keys = ("adapter", ("hash1", 2), "salt")
|
||||
assert to_event_extra_keys([extra_keys, None]) == [event_keys, None]
|
||||
assert to_event_extra_keys([]) is None
|
||||
assert to_event_extra_keys(None) is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize("hash_fn", [sha256, sha256_cbor])
|
||||
def test_hash_block_tokens(hash_fn):
|
||||
parent_block_hash = BlockHash(b"123")
|
||||
@@ -1013,9 +1059,11 @@ def test_request_block_hasher(hash_fn):
|
||||
block_hashes = request.block_hashes
|
||||
assert len(block_hashes) == 2
|
||||
assert block_hashes[0] == hash_fn(
|
||||
(kv_cache_utils.NONE_HASH, (0, 1, 2), (("hash1", 0),))
|
||||
(kv_cache_utils.NONE_HASH, (0, 1, 2), (("mm", "hash1", 0),))
|
||||
)
|
||||
assert block_hashes[1] == hash_fn(
|
||||
(block_hashes[0], (3, 4, 5), (("mm", "hash2", 0),))
|
||||
)
|
||||
assert block_hashes[1] == hash_fn((block_hashes[0], (3, 4, 5), (("hash2", 0),)))
|
||||
|
||||
|
||||
@pytest.mark.parametrize("hash_fn", [sha256, sha256_cbor])
|
||||
@@ -1043,7 +1091,7 @@ def test_request_block_hasher_incremental_append_with_multiple_mm_features(hash_
|
||||
)
|
||||
|
||||
expected_second_hash = hash_fn(
|
||||
(incremental.block_hashes[0], (4, 5, 6, 7), (("A", 0), ("B", 2)))
|
||||
(incremental.block_hashes[0], (4, 5, 6, 7), (("mm", "A", 0), ("mm", "B", 2)))
|
||||
)
|
||||
assert incremental.block_hashes[1] == expected_second_hash
|
||||
assert incremental.block_hashes == fresh.block_hashes
|
||||
@@ -3657,7 +3705,7 @@ def test_request_block_hasher_with_prompt_embeds(hash_fn: Callable[[Any], bytes]
|
||||
(
|
||||
kv_cache_utils.NONE_HASH,
|
||||
tuple(prompt_token_ids[:block_size]),
|
||||
(block1_embeds_hash,),
|
||||
(("prompt_embeds", block1_embeds_hash),),
|
||||
)
|
||||
)
|
||||
assert block_hashes[0] == expected_hash1
|
||||
@@ -3669,7 +3717,7 @@ def test_request_block_hasher_with_prompt_embeds(hash_fn: Callable[[Any], bytes]
|
||||
(
|
||||
block_hashes[0],
|
||||
tuple(prompt_token_ids[block_size:num_tokens]),
|
||||
(block2_embeds_hash,),
|
||||
(("prompt_embeds", block2_embeds_hash),),
|
||||
)
|
||||
)
|
||||
assert block_hashes[1] == expected_hash2
|
||||
@@ -3706,7 +3754,7 @@ def test_request_with_prompt_embeds_and_mm_inputs(hash_fn: Callable[[Any], bytes
|
||||
(
|
||||
kv_cache_utils.NONE_HASH,
|
||||
tuple(prompt_token_ids[:block_size]),
|
||||
(("hash1", 0), block1_embeds_hash),
|
||||
(("mm", "hash1", 0), ("prompt_embeds", block1_embeds_hash)),
|
||||
)
|
||||
)
|
||||
assert block_hashes[0] == expected_hash1
|
||||
@@ -3718,7 +3766,7 @@ def test_request_with_prompt_embeds_and_mm_inputs(hash_fn: Callable[[Any], bytes
|
||||
(
|
||||
block_hashes[0],
|
||||
tuple(prompt_token_ids[block_size:num_tokens]),
|
||||
(("hash2", 0), block2_embeds_hash),
|
||||
(("mm", "hash2", 0), ("prompt_embeds", block2_embeds_hash)),
|
||||
)
|
||||
)
|
||||
assert block_hashes[1] == expected_hash2
|
||||
|
||||
@@ -2901,21 +2901,21 @@ def test_mm_prefix_caching():
|
||||
(
|
||||
kv_cache_utils.NONE_HASH,
|
||||
tuple(all_token_ids[:block_size]),
|
||||
(("aaa", 11),),
|
||||
(("mm", "aaa", 11),),
|
||||
)
|
||||
)
|
||||
assert block_hashes[1] == sha256(
|
||||
(
|
||||
block_hashes[0],
|
||||
tuple(all_token_ids[block_size : block_size * 2]),
|
||||
(("aaa", -5), ("bbb", 14)),
|
||||
(("mm", "aaa", -5), ("mm", "bbb", 14)),
|
||||
)
|
||||
)
|
||||
assert block_hashes[2] == sha256(
|
||||
(
|
||||
block_hashes[1],
|
||||
tuple(all_token_ids[block_size * 2 : block_size * 3]),
|
||||
(("bbb", -2),),
|
||||
(("mm", "bbb", -2),),
|
||||
)
|
||||
)
|
||||
|
||||
@@ -2938,7 +2938,7 @@ def test_mm_prefix_caching():
|
||||
(
|
||||
block_hashes[2],
|
||||
tuple(all_token_ids[3 * block_size :] + [8] * 5),
|
||||
(("ccc", 0),),
|
||||
(("mm", "ccc", 0),),
|
||||
)
|
||||
)
|
||||
|
||||
@@ -2984,7 +2984,11 @@ def test_cache_key_salting():
|
||||
block_hashes = req0.block_hashes
|
||||
assert len(block_hashes) == 3
|
||||
assert block_hashes[0] == sha256(
|
||||
(kv_cache_utils.NONE_HASH, tuple(token_ids[:block_size]), ("salt1",))
|
||||
(
|
||||
kv_cache_utils.NONE_HASH,
|
||||
tuple(token_ids[:block_size]),
|
||||
(("cache_salt", "salt1"),),
|
||||
)
|
||||
)
|
||||
assert block_hashes[1] == sha256(
|
||||
(block_hashes[0], tuple(token_ids[block_size : block_size * 2]), None)
|
||||
@@ -3029,7 +3033,11 @@ def test_cache_key_salting():
|
||||
block_hashes = req2.block_hashes
|
||||
assert len(block_hashes) == 3
|
||||
assert block_hashes[0] == sha256(
|
||||
(kv_cache_utils.NONE_HASH, tuple(token_ids[:block_size]), ("salt2",))
|
||||
(
|
||||
kv_cache_utils.NONE_HASH,
|
||||
tuple(token_ids[:block_size]),
|
||||
(("cache_salt", "salt2"),),
|
||||
)
|
||||
)
|
||||
assert block_hashes[1] == sha256(
|
||||
(block_hashes[0], tuple(token_ids[block_size : block_size * 2]), None)
|
||||
|
||||
Reference in New Issue
Block a user