From 765872e7ed5b6f372f0898a67e48d3365fa28139 Mon Sep 17 00:00:00 2001 From: Clinton Thomas <1033162+KernelClint@users.noreply.github.com> Date: Tue, 29 Sep 2026 21:24:59 -0400 Subject: [PATCH] [Core][BugFix] Tag prefix-cache extra keys by source (#51899) Signed-off-by: Nick Hill Co-authored-by: Lucas Bourtoule <35483370+dhalf@users.noreply.github.com> Co-authored-by: Tai An Co-authored-by: Nick Hill Co-authored-by: Claude Opus 5.5 --- tests/v1/core/test_kv_cache_utils.py | 80 +++++++++++++++---- tests/v1/core/test_prefix_caching.py | 20 +++-- vllm/distributed/kv_events.py | 8 +- .../kv_connector/v1/offloading/events.py | 5 +- vllm/v1/core/block_pool.py | 5 +- vllm/v1/core/kv_cache_utils.py | 51 +++++++++--- vllm/v1/simple_kv_offload/manager.py | 5 +- 7 files changed, 131 insertions(+), 43 deletions(-) diff --git a/tests/v1/core/test_kv_cache_utils.py b/tests/v1/core/test_kv_cache_utils.py index c2c86053f0..478707af9d 100644 --- a/tests/v1/core/test_kv_cache_utils.py +++ b/tests/v1/core/test_kv_cache_utils.py @@ -54,6 +54,7 @@ from vllm.v1.core.kv_cache_utils import ( is_kv_cache_spec_uniform, make_block_hash_with_group_id, tensor_data, + to_event_extra_keys, ) from vllm.v1.hisparse.layout import ( create_hisparse_layout, @@ -822,12 +823,12 @@ def test_generate_block_hash_extra_keys(): # Test with no extra keys extra_keys, next_mm_idx = generate_block_hash_extra_keys(request, 0, 5, 0) - assert extra_keys == (("hash1", 0),) + assert extra_keys == (("mm", "hash1", 0),) assert next_mm_idx == 1 # Test with partial overlap extra_keys, next_mm_idx = generate_block_hash_extra_keys(request, 3, 8, 0) - assert extra_keys == (("hash1", -3),) + assert extra_keys == (("mm", "hash1", -3),) assert next_mm_idx == 1 # Test with no overlap @@ -837,7 +838,7 @@ def test_generate_block_hash_extra_keys(): # Test with multiple extra keys extra_keys, next_mm_idx = generate_block_hash_extra_keys(request, 0, 15, 0) - assert extra_keys == (("hash1", 0), ("hash2", 10)) + assert extra_keys == (("mm", "hash1", 0), ("mm", "hash2", 10)) assert next_mm_idx == 2 @@ -865,9 +866,9 @@ def test_generate_block_hash_extra_keys_cache_salt(): # salt is added for the first token extra_keys, _ = generate_block_hash_extra_keys(request, 0, 1, 0) - assert extra_keys == ("salt",) + assert extra_keys == (("cache_salt", "salt"),) extra_keys, _ = generate_block_hash_extra_keys(request, 0, 10, 0) - assert extra_keys == ("salt",) + assert extra_keys == (("cache_salt", "salt"),) # no salt added for other tokens extra_keys, _ = generate_block_hash_extra_keys(request, 1, 2, 0) @@ -888,7 +889,7 @@ def test_generate_block_hash_extra_keys_cache_salt(): # Test with no extra keys extra_keys, next_mm_idx = generate_block_hash_extra_keys(request_mm, 0, 5, 0) - assert extra_keys == (("hash1", 0), "salt") + assert extra_keys == (("mm", "hash1", 0), ("cache_salt", "salt")) assert next_mm_idx == 1 @@ -906,13 +907,13 @@ def test_generate_block_hash_extra_keys_prompt_embeds(): extra_keys, _ = generate_block_hash_extra_keys(request, 0, 5, 0) expected_embeds = prompt_embeds[0:5] expected_hash = hashlib.sha256(kv_cache_utils.tensor_data(expected_embeds)).digest() - assert extra_keys == (expected_hash,) + assert extra_keys == (("prompt_embeds", expected_hash),) # Test with prompt embeds for the second block extra_keys, _ = generate_block_hash_extra_keys(request, 5, 10, 0) expected_embeds = prompt_embeds[5:10] expected_hash = hashlib.sha256(kv_cache_utils.tensor_data(expected_embeds)).digest() - assert extra_keys == (expected_hash,) + assert extra_keys == (("prompt_embeds", expected_hash),) def test_generate_block_hash_extra_keys_prompt_embeds_cached(monkeypatch): @@ -976,13 +977,58 @@ def test_generate_block_hash_extra_keys_lora(): ) extra_keys, _ = generate_block_hash_extra_keys(request, 0, 3, 0) - assert extra_keys == ("test_lora_adapter",) + assert extra_keys == (("lora", "test_lora_adapter"),) request.lora_request = None extra_keys, _ = generate_block_hash_extra_keys(request, 0, 3, 0) assert extra_keys is None +@pytest.mark.parametrize("hash_fn", [sha256, sha256_cbor]) +def test_lora_name_and_cache_salt_block_hashes_do_not_collide(hash_fn): + """A cache_salt equal to a LoRA name must not share that LoRA's blocks.""" + lora_request = LoRARequest( + lora_name="foo", lora_int_id=1, lora_path="/path/to/lora" + ) + lora_req = Request( + request_id="lora", + prompt_token_ids=[0, 1, 2], + sampling_params=SamplingParams(max_tokens=1), + pooling_params=None, + lora_request=lora_request, + block_hasher=get_request_block_hasher(3, hash_fn), + ) + salted_req = make_request("salted", [0, 1, 2], hash_fn=hash_fn, cache_salt="foo") + + assert lora_req.block_hashes[0] != salted_req.block_hashes[0] + + +def test_to_event_extra_keys_keeps_untagged_event_format(): + """KV events keep publishing the extra-key shapes consumers already parse.""" + request = make_request( + "0", + list(range(10)), + mm_positions=[PlaceholderRange(offset=2, length=3)], + mm_hashes=["hash1"], + cache_salt="salt", + ) + request.lora_request = LoRARequest( + lora_name="adapter", lora_int_id=1, lora_path="/path/to/lora" + ) + + extra_keys, _ = generate_block_hash_extra_keys(request, 0, 5, 0) + + assert extra_keys == ( + ("lora", "adapter"), + ("mm", "hash1", 2), + ("cache_salt", "salt"), + ) + event_keys = ("adapter", ("hash1", 2), "salt") + assert to_event_extra_keys([extra_keys, None]) == [event_keys, None] + assert to_event_extra_keys([]) is None + assert to_event_extra_keys(None) is None + + @pytest.mark.parametrize("hash_fn", [sha256, sha256_cbor]) def test_hash_block_tokens(hash_fn): parent_block_hash = BlockHash(b"123") @@ -1013,9 +1059,11 @@ def test_request_block_hasher(hash_fn): block_hashes = request.block_hashes assert len(block_hashes) == 2 assert block_hashes[0] == hash_fn( - (kv_cache_utils.NONE_HASH, (0, 1, 2), (("hash1", 0),)) + (kv_cache_utils.NONE_HASH, (0, 1, 2), (("mm", "hash1", 0),)) + ) + assert block_hashes[1] == hash_fn( + (block_hashes[0], (3, 4, 5), (("mm", "hash2", 0),)) ) - assert block_hashes[1] == hash_fn((block_hashes[0], (3, 4, 5), (("hash2", 0),))) @pytest.mark.parametrize("hash_fn", [sha256, sha256_cbor]) @@ -1043,7 +1091,7 @@ def test_request_block_hasher_incremental_append_with_multiple_mm_features(hash_ ) expected_second_hash = hash_fn( - (incremental.block_hashes[0], (4, 5, 6, 7), (("A", 0), ("B", 2))) + (incremental.block_hashes[0], (4, 5, 6, 7), (("mm", "A", 0), ("mm", "B", 2))) ) assert incremental.block_hashes[1] == expected_second_hash assert incremental.block_hashes == fresh.block_hashes @@ -3657,7 +3705,7 @@ def test_request_block_hasher_with_prompt_embeds(hash_fn: Callable[[Any], bytes] ( kv_cache_utils.NONE_HASH, tuple(prompt_token_ids[:block_size]), - (block1_embeds_hash,), + (("prompt_embeds", block1_embeds_hash),), ) ) assert block_hashes[0] == expected_hash1 @@ -3669,7 +3717,7 @@ def test_request_block_hasher_with_prompt_embeds(hash_fn: Callable[[Any], bytes] ( block_hashes[0], tuple(prompt_token_ids[block_size:num_tokens]), - (block2_embeds_hash,), + (("prompt_embeds", block2_embeds_hash),), ) ) assert block_hashes[1] == expected_hash2 @@ -3706,7 +3754,7 @@ def test_request_with_prompt_embeds_and_mm_inputs(hash_fn: Callable[[Any], bytes ( kv_cache_utils.NONE_HASH, tuple(prompt_token_ids[:block_size]), - (("hash1", 0), block1_embeds_hash), + (("mm", "hash1", 0), ("prompt_embeds", block1_embeds_hash)), ) ) assert block_hashes[0] == expected_hash1 @@ -3718,7 +3766,7 @@ def test_request_with_prompt_embeds_and_mm_inputs(hash_fn: Callable[[Any], bytes ( block_hashes[0], tuple(prompt_token_ids[block_size:num_tokens]), - (("hash2", 0), block2_embeds_hash), + (("mm", "hash2", 0), ("prompt_embeds", block2_embeds_hash)), ) ) assert block_hashes[1] == expected_hash2 diff --git a/tests/v1/core/test_prefix_caching.py b/tests/v1/core/test_prefix_caching.py index 99d3235006..c46c179260 100644 --- a/tests/v1/core/test_prefix_caching.py +++ b/tests/v1/core/test_prefix_caching.py @@ -2901,21 +2901,21 @@ def test_mm_prefix_caching(): ( kv_cache_utils.NONE_HASH, tuple(all_token_ids[:block_size]), - (("aaa", 11),), + (("mm", "aaa", 11),), ) ) assert block_hashes[1] == sha256( ( block_hashes[0], tuple(all_token_ids[block_size : block_size * 2]), - (("aaa", -5), ("bbb", 14)), + (("mm", "aaa", -5), ("mm", "bbb", 14)), ) ) assert block_hashes[2] == sha256( ( block_hashes[1], tuple(all_token_ids[block_size * 2 : block_size * 3]), - (("bbb", -2),), + (("mm", "bbb", -2),), ) ) @@ -2938,7 +2938,7 @@ def test_mm_prefix_caching(): ( block_hashes[2], tuple(all_token_ids[3 * block_size :] + [8] * 5), - (("ccc", 0),), + (("mm", "ccc", 0),), ) ) @@ -2984,7 +2984,11 @@ def test_cache_key_salting(): block_hashes = req0.block_hashes assert len(block_hashes) == 3 assert block_hashes[0] == sha256( - (kv_cache_utils.NONE_HASH, tuple(token_ids[:block_size]), ("salt1",)) + ( + kv_cache_utils.NONE_HASH, + tuple(token_ids[:block_size]), + (("cache_salt", "salt1"),), + ) ) assert block_hashes[1] == sha256( (block_hashes[0], tuple(token_ids[block_size : block_size * 2]), None) @@ -3029,7 +3033,11 @@ def test_cache_key_salting(): block_hashes = req2.block_hashes assert len(block_hashes) == 3 assert block_hashes[0] == sha256( - (kv_cache_utils.NONE_HASH, tuple(token_ids[:block_size]), ("salt2",)) + ( + kv_cache_utils.NONE_HASH, + tuple(token_ids[:block_size]), + (("cache_salt", "salt2"),), + ) ) assert block_hashes[1] == sha256( (block_hashes[0], tuple(token_ids[block_size : block_size * 2]), None) diff --git a/vllm/distributed/kv_events.py b/vllm/distributed/kv_events.py index 9d7ec2f827..9eb0805b7b 100644 --- a/vllm/distributed/kv_events.py +++ b/vllm/distributed/kv_events.py @@ -68,9 +68,11 @@ class BlockStored(KVCacheEvent): lora_name: str | None """Extra keys used in block hash computation, one entry per block in - block_hashes. Each entry contains MM identifiers, LoRA name, cache_salt, - prompt embedding hashes, etc. for that specific block. Exposed for external - KV cache consumers to reconstruct block hashes. + block_hashes. Each entry contains, in order, the LoRA name (if + `lora_name` is set), `(mm_identifier, offset)` pairs, the cache_salt + string and prompt embedding digests for that specific block. These are + published untagged; the block hash itself uses each value tagged with + its source, e.g. `("lora", name)` or `("mm", mm_identifier, offset)`. """ extra_keys: list[tuple[Any, ...] | None] | None = None diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/offloading/events.py b/vllm/distributed/kv_transfer/kv_connector/v1/offloading/events.py index 68318a16da..05609fe86f 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/offloading/events.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/offloading/events.py @@ -31,6 +31,7 @@ from vllm.v1.core.kv_cache_utils import ( BlockHash, maybe_convert_block_hash, resolve_block_hashes, + to_event_extra_keys, ) from vllm.v1.kv_cache_interface import ( KVCacheGroupSpec, @@ -393,9 +394,7 @@ class OffloadingEventsTracker: lora_id=meta.lora_id, medium=_MEDIUM_TO_EVENT_STR[event.medium], lora_name=meta.lora_name, - extra_keys=( - list(meta.extra_keys) if meta.extra_keys is not None else None - ), + extra_keys=to_event_extra_keys(meta.extra_keys), group_idx=meta.group_idx, kv_cache_spec_kind=meta.kv_cache_spec.kv_cache_spec_kind, kv_cache_spec_sliding_window=( diff --git a/vllm/v1/core/block_pool.py b/vllm/v1/core/block_pool.py index 9289e7c1f8..664804d50d 100644 --- a/vllm/v1/core/block_pool.py +++ b/vllm/v1/core/block_pool.py @@ -25,6 +25,7 @@ from vllm.v1.core.kv_cache_utils import ( make_block_hash_with_group_id, maybe_convert_block_hash, resolve_block_hashes, + to_event_extra_keys, ) from vllm.v1.request import Request @@ -367,7 +368,7 @@ class BlockPool: lora_id=request.lora_request.adapter_id if request.lora_request else None, medium=self.medium, lora_name=request.lora_request.name if request.lora_request else None, - extra_keys=extra_keys_list if extra_keys_list else None, + extra_keys=to_event_extra_keys(extra_keys_list), group_idx=kv_cache_group_id, session_id=request.session_id, ) @@ -557,7 +558,7 @@ class BlockPool: lora_name=request.lora_request.name if request.lora_request else None, - extra_keys=[extra_keys], + extra_keys=to_event_extra_keys([extra_keys]), group_idx=kv_cache_group_id, session_id=request.session_id, ) diff --git a/vllm/v1/core/kv_cache_utils.py b/vllm/v1/core/kv_cache_utils.py index ddeda3ed19..7ef71dd471 100644 --- a/vllm/v1/core/kv_cache_utils.py +++ b/vllm/v1/core/kv_cache_utils.py @@ -502,8 +502,8 @@ def _gen_mm_extra_hash_keys( ) -> tuple[list[Any], int]: """Generate extra keys related to MultiModal request for block hash computation. For multi-modal inputs, the extra keys are - (mm_hash, start_offset) that indicate a mm input contained in the - block and its starting offset in the block tokens. + ("mm", mm_hash, start_offset) tuples that indicate a mm input contained + in the block and its starting offset in the block tokens. Args: request: The request object. @@ -549,7 +549,7 @@ def _gen_mm_extra_hash_keys( # relative to the start of the block so prefix-cache keys stay # distinct when the same MM item appears at different positions # within otherwise-identical placeholder blocks. - extra_keys.append((mm_feature.identifier, offset - start_token_idx)) + extra_keys.append(("mm", mm_feature.identifier, offset - start_token_idx)) if end_token_idx >= offset + length: # If this block contains the end of the current mm input, @@ -565,7 +565,7 @@ def _gen_mm_extra_hash_keys( return extra_keys, curr_mm_idx -def _gen_lora_extra_hash_keys(request: Request) -> list[str]: +def _gen_lora_extra_hash_keys(request: Request) -> list[tuple[str, str]]: """Generate extra keys related to LoRA for block hash computation. Args: @@ -578,12 +578,12 @@ def _gen_lora_extra_hash_keys(request: Request) -> list[str]: """ if not request.lora_request: return [] - return [request.lora_request.lora_name] + return [("lora", request.lora_request.lora_name)] def _gen_prompt_embeds_extra_hash_keys( request: Request, start_token_idx: int, end_token_idx: int -) -> list[bytes]: +) -> list[tuple[str, bytes]]: """Generate extra keys related to prompt embeds for block hash computation. Args: @@ -605,7 +605,7 @@ def _gen_prompt_embeds_extra_hash_keys( # Hash prompt embeds once per block and cache on request embeds_hash = hashlib.sha256(tensor_data(block_prompt_embeds)).digest() request._prompt_embeds_per_block_hashes[block_range] = embeds_hash - return [embeds_hash] + return [("prompt_embeds", embeds_hash)] def generate_block_hash_extra_keys( @@ -629,9 +629,11 @@ def generate_block_hash_extra_keys( mm_extra_keys, new_start_mm_idx = _gen_mm_extra_hash_keys( request, start_token_idx, end_token_idx, start_mm_idx ) - lora_extra_keys: list[str] = _gen_lora_extra_hash_keys(request) - cache_salt_keys: list[str] = ( - [request.cache_salt] if (start_token_idx == 0 and request.cache_salt) else [] + lora_extra_keys = _gen_lora_extra_hash_keys(request) + cache_salt_keys: list[tuple[str, str]] = ( + [("cache_salt", request.cache_salt)] + if (start_token_idx == 0 and request.cache_salt) + else [] ) prompt_embeds_keys = _gen_prompt_embeds_extra_hash_keys( request, start_token_idx, end_token_idx @@ -647,6 +649,35 @@ def generate_block_hash_extra_keys( return tuple(extra_keys), new_start_mm_idx +def to_event_extra_keys( + extra_keys: Iterable[tuple[Any, ...] | None] | None, +) -> list[tuple[Any, ...] | None] | None: + """Convert block-hash extra keys to the untagged per-block list published + in KV events. + + External KV event consumers parse the pre-tagging shapes: bare LoRA names + and cache salts, ``(mm_identifier, offset)`` pairs and bare prompt-embeds + digests. Events keep that format until consumers handle the tagged keys. + + Args: + extra_keys: One entry per block, each as returned by + `generate_block_hash_extra_keys`, or None. + + Returns: + One untagged entry per block, or None if there are no entries. + + """ + if not extra_keys: + return None + event_keys = [ + None + if keys is None + else tuple(key[1:] if key[0] == "mm" else key[1] for key in keys) + for keys in extra_keys + ] + return event_keys or None + + def hash_block_tokens( hash_function: Callable[[Any], bytes], parent_block_hash: BlockHash | None, diff --git a/vllm/v1/simple_kv_offload/manager.py b/vllm/v1/simple_kv_offload/manager.py index f70afaa343..c91537375f 100644 --- a/vllm/v1/simple_kv_offload/manager.py +++ b/vllm/v1/simple_kv_offload/manager.py @@ -34,6 +34,7 @@ from vllm.v1.core.kv_cache_utils import ( maybe_convert_block_hash, resolve_block_hashes, resolve_dcp_kv_block_size, + to_event_extra_keys, ) from vllm.v1.core.sched.output import SchedulerOutput from vllm.v1.kv_cache_interface import ( @@ -1147,9 +1148,7 @@ class SimpleCPUOffloadScheduler: lora_id=meta.lora_id if meta else None, medium=self.kv_event_medium, lora_name=meta.lora_name if meta else None, - extra_keys=( - [extra_keys] if extra_keys is not None else None - ), + extra_keys=to_event_extra_keys(extra_keys and [extra_keys]), group_idx=group_idx, kv_cache_spec_kind=get_kv_cache_spec_kind(spec).value, kv_cache_spec_sliding_window=(