[Core][BugFix] Tag prefix-cache extra keys by source (#51899)

Signed-off-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: Lucas Bourtoule <35483370+dhalf@users.noreply.github.com>
Co-authored-by: Tai An <antai12232931@outlook.com>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Clinton Thomas
2026-09-29 18:24:59 -07:00
committed by GitHub
co-authored by Lucas Bourtoule Tai An Nick Hill Claude Opus 5.5
parent 09c47db1ca
commit 765872e7ed
7 changed files with 131 additions and 43 deletions
+64 -16
View File
@@ -54,6 +54,7 @@ from vllm.v1.core.kv_cache_utils import (
is_kv_cache_spec_uniform,
make_block_hash_with_group_id,
tensor_data,
to_event_extra_keys,
)
from vllm.v1.hisparse.layout import (
create_hisparse_layout,
@@ -822,12 +823,12 @@ def test_generate_block_hash_extra_keys():
# Test with no extra keys
extra_keys, next_mm_idx = generate_block_hash_extra_keys(request, 0, 5, 0)
assert extra_keys == (("hash1", 0),)
assert extra_keys == (("mm", "hash1", 0),)
assert next_mm_idx == 1
# Test with partial overlap
extra_keys, next_mm_idx = generate_block_hash_extra_keys(request, 3, 8, 0)
assert extra_keys == (("hash1", -3),)
assert extra_keys == (("mm", "hash1", -3),)
assert next_mm_idx == 1
# Test with no overlap
@@ -837,7 +838,7 @@ def test_generate_block_hash_extra_keys():
# Test with multiple extra keys
extra_keys, next_mm_idx = generate_block_hash_extra_keys(request, 0, 15, 0)
assert extra_keys == (("hash1", 0), ("hash2", 10))
assert extra_keys == (("mm", "hash1", 0), ("mm", "hash2", 10))
assert next_mm_idx == 2
@@ -865,9 +866,9 @@ def test_generate_block_hash_extra_keys_cache_salt():
# salt is added for the first token
extra_keys, _ = generate_block_hash_extra_keys(request, 0, 1, 0)
assert extra_keys == ("salt",)
assert extra_keys == (("cache_salt", "salt"),)
extra_keys, _ = generate_block_hash_extra_keys(request, 0, 10, 0)
assert extra_keys == ("salt",)
assert extra_keys == (("cache_salt", "salt"),)
# no salt added for other tokens
extra_keys, _ = generate_block_hash_extra_keys(request, 1, 2, 0)
@@ -888,7 +889,7 @@ def test_generate_block_hash_extra_keys_cache_salt():
# Test with no extra keys
extra_keys, next_mm_idx = generate_block_hash_extra_keys(request_mm, 0, 5, 0)
assert extra_keys == (("hash1", 0), "salt")
assert extra_keys == (("mm", "hash1", 0), ("cache_salt", "salt"))
assert next_mm_idx == 1
@@ -906,13 +907,13 @@ def test_generate_block_hash_extra_keys_prompt_embeds():
extra_keys, _ = generate_block_hash_extra_keys(request, 0, 5, 0)
expected_embeds = prompt_embeds[0:5]
expected_hash = hashlib.sha256(kv_cache_utils.tensor_data(expected_embeds)).digest()
assert extra_keys == (expected_hash,)
assert extra_keys == (("prompt_embeds", expected_hash),)
# Test with prompt embeds for the second block
extra_keys, _ = generate_block_hash_extra_keys(request, 5, 10, 0)
expected_embeds = prompt_embeds[5:10]
expected_hash = hashlib.sha256(kv_cache_utils.tensor_data(expected_embeds)).digest()
assert extra_keys == (expected_hash,)
assert extra_keys == (("prompt_embeds", expected_hash),)
def test_generate_block_hash_extra_keys_prompt_embeds_cached(monkeypatch):
@@ -976,13 +977,58 @@ def test_generate_block_hash_extra_keys_lora():
)
extra_keys, _ = generate_block_hash_extra_keys(request, 0, 3, 0)
assert extra_keys == ("test_lora_adapter",)
assert extra_keys == (("lora", "test_lora_adapter"),)
request.lora_request = None
extra_keys, _ = generate_block_hash_extra_keys(request, 0, 3, 0)
assert extra_keys is None
@pytest.mark.parametrize("hash_fn", [sha256, sha256_cbor])
def test_lora_name_and_cache_salt_block_hashes_do_not_collide(hash_fn):
"""A cache_salt equal to a LoRA name must not share that LoRA's blocks."""
lora_request = LoRARequest(
lora_name="foo", lora_int_id=1, lora_path="/path/to/lora"
)
lora_req = Request(
request_id="lora",
prompt_token_ids=[0, 1, 2],
sampling_params=SamplingParams(max_tokens=1),
pooling_params=None,
lora_request=lora_request,
block_hasher=get_request_block_hasher(3, hash_fn),
)
salted_req = make_request("salted", [0, 1, 2], hash_fn=hash_fn, cache_salt="foo")
assert lora_req.block_hashes[0] != salted_req.block_hashes[0]
def test_to_event_extra_keys_keeps_untagged_event_format():
"""KV events keep publishing the extra-key shapes consumers already parse."""
request = make_request(
"0",
list(range(10)),
mm_positions=[PlaceholderRange(offset=2, length=3)],
mm_hashes=["hash1"],
cache_salt="salt",
)
request.lora_request = LoRARequest(
lora_name="adapter", lora_int_id=1, lora_path="/path/to/lora"
)
extra_keys, _ = generate_block_hash_extra_keys(request, 0, 5, 0)
assert extra_keys == (
("lora", "adapter"),
("mm", "hash1", 2),
("cache_salt", "salt"),
)
event_keys = ("adapter", ("hash1", 2), "salt")
assert to_event_extra_keys([extra_keys, None]) == [event_keys, None]
assert to_event_extra_keys([]) is None
assert to_event_extra_keys(None) is None
@pytest.mark.parametrize("hash_fn", [sha256, sha256_cbor])
def test_hash_block_tokens(hash_fn):
parent_block_hash = BlockHash(b"123")
@@ -1013,9 +1059,11 @@ def test_request_block_hasher(hash_fn):
block_hashes = request.block_hashes
assert len(block_hashes) == 2
assert block_hashes[0] == hash_fn(
(kv_cache_utils.NONE_HASH, (0, 1, 2), (("hash1", 0),))
(kv_cache_utils.NONE_HASH, (0, 1, 2), (("mm", "hash1", 0),))
)
assert block_hashes[1] == hash_fn(
(block_hashes[0], (3, 4, 5), (("mm", "hash2", 0),))
)
assert block_hashes[1] == hash_fn((block_hashes[0], (3, 4, 5), (("hash2", 0),)))
@pytest.mark.parametrize("hash_fn", [sha256, sha256_cbor])
@@ -1043,7 +1091,7 @@ def test_request_block_hasher_incremental_append_with_multiple_mm_features(hash_
)
expected_second_hash = hash_fn(
(incremental.block_hashes[0], (4, 5, 6, 7), (("A", 0), ("B", 2)))
(incremental.block_hashes[0], (4, 5, 6, 7), (("mm", "A", 0), ("mm", "B", 2)))
)
assert incremental.block_hashes[1] == expected_second_hash
assert incremental.block_hashes == fresh.block_hashes
@@ -3657,7 +3705,7 @@ def test_request_block_hasher_with_prompt_embeds(hash_fn: Callable[[Any], bytes]
(
kv_cache_utils.NONE_HASH,
tuple(prompt_token_ids[:block_size]),
(block1_embeds_hash,),
(("prompt_embeds", block1_embeds_hash),),
)
)
assert block_hashes[0] == expected_hash1
@@ -3669,7 +3717,7 @@ def test_request_block_hasher_with_prompt_embeds(hash_fn: Callable[[Any], bytes]
(
block_hashes[0],
tuple(prompt_token_ids[block_size:num_tokens]),
(block2_embeds_hash,),
(("prompt_embeds", block2_embeds_hash),),
)
)
assert block_hashes[1] == expected_hash2
@@ -3706,7 +3754,7 @@ def test_request_with_prompt_embeds_and_mm_inputs(hash_fn: Callable[[Any], bytes
(
kv_cache_utils.NONE_HASH,
tuple(prompt_token_ids[:block_size]),
(("hash1", 0), block1_embeds_hash),
(("mm", "hash1", 0), ("prompt_embeds", block1_embeds_hash)),
)
)
assert block_hashes[0] == expected_hash1
@@ -3718,7 +3766,7 @@ def test_request_with_prompt_embeds_and_mm_inputs(hash_fn: Callable[[Any], bytes
(
block_hashes[0],
tuple(prompt_token_ids[block_size:num_tokens]),
(("hash2", 0), block2_embeds_hash),
(("mm", "hash2", 0), ("prompt_embeds", block2_embeds_hash)),
)
)
assert block_hashes[1] == expected_hash2
+14 -6
View File
@@ -2901,21 +2901,21 @@ def test_mm_prefix_caching():
(
kv_cache_utils.NONE_HASH,
tuple(all_token_ids[:block_size]),
(("aaa", 11),),
(("mm", "aaa", 11),),
)
)
assert block_hashes[1] == sha256(
(
block_hashes[0],
tuple(all_token_ids[block_size : block_size * 2]),
(("aaa", -5), ("bbb", 14)),
(("mm", "aaa", -5), ("mm", "bbb", 14)),
)
)
assert block_hashes[2] == sha256(
(
block_hashes[1],
tuple(all_token_ids[block_size * 2 : block_size * 3]),
(("bbb", -2),),
(("mm", "bbb", -2),),
)
)
@@ -2938,7 +2938,7 @@ def test_mm_prefix_caching():
(
block_hashes[2],
tuple(all_token_ids[3 * block_size :] + [8] * 5),
(("ccc", 0),),
(("mm", "ccc", 0),),
)
)
@@ -2984,7 +2984,11 @@ def test_cache_key_salting():
block_hashes = req0.block_hashes
assert len(block_hashes) == 3
assert block_hashes[0] == sha256(
(kv_cache_utils.NONE_HASH, tuple(token_ids[:block_size]), ("salt1",))
(
kv_cache_utils.NONE_HASH,
tuple(token_ids[:block_size]),
(("cache_salt", "salt1"),),
)
)
assert block_hashes[1] == sha256(
(block_hashes[0], tuple(token_ids[block_size : block_size * 2]), None)
@@ -3029,7 +3033,11 @@ def test_cache_key_salting():
block_hashes = req2.block_hashes
assert len(block_hashes) == 3
assert block_hashes[0] == sha256(
(kv_cache_utils.NONE_HASH, tuple(token_ids[:block_size]), ("salt2",))
(
kv_cache_utils.NONE_HASH,
tuple(token_ids[:block_size]),
(("cache_salt", "salt2"),),
)
)
assert block_hashes[1] == sha256(
(block_hashes[0], tuple(token_ids[block_size : block_size * 2]), None)