[Core][BugFix] Tag prefix-cache extra keys by source (#51899)

Signed-off-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: Lucas Bourtoule <35483370+dhalf@users.noreply.github.com>
Co-authored-by: Tai An <antai12232931@outlook.com>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Clinton Thomas
2026-09-29 18:24:59 -07:00
committed by GitHub
co-authored by Lucas Bourtoule Tai An Nick Hill Claude Opus 5.5
parent 09c47db1ca
commit 765872e7ed
7 changed files with 131 additions and 43 deletions
+64 -16
View File
@@ -54,6 +54,7 @@ from vllm.v1.core.kv_cache_utils import (
is_kv_cache_spec_uniform,
make_block_hash_with_group_id,
tensor_data,
to_event_extra_keys,
)
from vllm.v1.hisparse.layout import (
create_hisparse_layout,
@@ -822,12 +823,12 @@ def test_generate_block_hash_extra_keys():
# Test with no extra keys
extra_keys, next_mm_idx = generate_block_hash_extra_keys(request, 0, 5, 0)
assert extra_keys == (("hash1", 0),)
assert extra_keys == (("mm", "hash1", 0),)
assert next_mm_idx == 1
# Test with partial overlap
extra_keys, next_mm_idx = generate_block_hash_extra_keys(request, 3, 8, 0)
assert extra_keys == (("hash1", -3),)
assert extra_keys == (("mm", "hash1", -3),)
assert next_mm_idx == 1
# Test with no overlap
@@ -837,7 +838,7 @@ def test_generate_block_hash_extra_keys():
# Test with multiple extra keys
extra_keys, next_mm_idx = generate_block_hash_extra_keys(request, 0, 15, 0)
assert extra_keys == (("hash1", 0), ("hash2", 10))
assert extra_keys == (("mm", "hash1", 0), ("mm", "hash2", 10))
assert next_mm_idx == 2
@@ -865,9 +866,9 @@ def test_generate_block_hash_extra_keys_cache_salt():
# salt is added for the first token
extra_keys, _ = generate_block_hash_extra_keys(request, 0, 1, 0)
assert extra_keys == ("salt",)
assert extra_keys == (("cache_salt", "salt"),)
extra_keys, _ = generate_block_hash_extra_keys(request, 0, 10, 0)
assert extra_keys == ("salt",)
assert extra_keys == (("cache_salt", "salt"),)
# no salt added for other tokens
extra_keys, _ = generate_block_hash_extra_keys(request, 1, 2, 0)
@@ -888,7 +889,7 @@ def test_generate_block_hash_extra_keys_cache_salt():
# Test with no extra keys
extra_keys, next_mm_idx = generate_block_hash_extra_keys(request_mm, 0, 5, 0)
assert extra_keys == (("hash1", 0), "salt")
assert extra_keys == (("mm", "hash1", 0), ("cache_salt", "salt"))
assert next_mm_idx == 1
@@ -906,13 +907,13 @@ def test_generate_block_hash_extra_keys_prompt_embeds():
extra_keys, _ = generate_block_hash_extra_keys(request, 0, 5, 0)
expected_embeds = prompt_embeds[0:5]
expected_hash = hashlib.sha256(kv_cache_utils.tensor_data(expected_embeds)).digest()
assert extra_keys == (expected_hash,)
assert extra_keys == (("prompt_embeds", expected_hash),)
# Test with prompt embeds for the second block
extra_keys, _ = generate_block_hash_extra_keys(request, 5, 10, 0)
expected_embeds = prompt_embeds[5:10]
expected_hash = hashlib.sha256(kv_cache_utils.tensor_data(expected_embeds)).digest()
assert extra_keys == (expected_hash,)
assert extra_keys == (("prompt_embeds", expected_hash),)
def test_generate_block_hash_extra_keys_prompt_embeds_cached(monkeypatch):
@@ -976,13 +977,58 @@ def test_generate_block_hash_extra_keys_lora():
)
extra_keys, _ = generate_block_hash_extra_keys(request, 0, 3, 0)
assert extra_keys == ("test_lora_adapter",)
assert extra_keys == (("lora", "test_lora_adapter"),)
request.lora_request = None
extra_keys, _ = generate_block_hash_extra_keys(request, 0, 3, 0)
assert extra_keys is None
@pytest.mark.parametrize("hash_fn", [sha256, sha256_cbor])
def test_lora_name_and_cache_salt_block_hashes_do_not_collide(hash_fn):
"""A cache_salt equal to a LoRA name must not share that LoRA's blocks."""
lora_request = LoRARequest(
lora_name="foo", lora_int_id=1, lora_path="/path/to/lora"
)
lora_req = Request(
request_id="lora",
prompt_token_ids=[0, 1, 2],
sampling_params=SamplingParams(max_tokens=1),
pooling_params=None,
lora_request=lora_request,
block_hasher=get_request_block_hasher(3, hash_fn),
)
salted_req = make_request("salted", [0, 1, 2], hash_fn=hash_fn, cache_salt="foo")
assert lora_req.block_hashes[0] != salted_req.block_hashes[0]
def test_to_event_extra_keys_keeps_untagged_event_format():
"""KV events keep publishing the extra-key shapes consumers already parse."""
request = make_request(
"0",
list(range(10)),
mm_positions=[PlaceholderRange(offset=2, length=3)],
mm_hashes=["hash1"],
cache_salt="salt",
)
request.lora_request = LoRARequest(
lora_name="adapter", lora_int_id=1, lora_path="/path/to/lora"
)
extra_keys, _ = generate_block_hash_extra_keys(request, 0, 5, 0)
assert extra_keys == (
("lora", "adapter"),
("mm", "hash1", 2),
("cache_salt", "salt"),
)
event_keys = ("adapter", ("hash1", 2), "salt")
assert to_event_extra_keys([extra_keys, None]) == [event_keys, None]
assert to_event_extra_keys([]) is None
assert to_event_extra_keys(None) is None
@pytest.mark.parametrize("hash_fn", [sha256, sha256_cbor])
def test_hash_block_tokens(hash_fn):
parent_block_hash = BlockHash(b"123")
@@ -1013,9 +1059,11 @@ def test_request_block_hasher(hash_fn):
block_hashes = request.block_hashes
assert len(block_hashes) == 2
assert block_hashes[0] == hash_fn(
(kv_cache_utils.NONE_HASH, (0, 1, 2), (("hash1", 0),))
(kv_cache_utils.NONE_HASH, (0, 1, 2), (("mm", "hash1", 0),))
)
assert block_hashes[1] == hash_fn(
(block_hashes[0], (3, 4, 5), (("mm", "hash2", 0),))
)
assert block_hashes[1] == hash_fn((block_hashes[0], (3, 4, 5), (("hash2", 0),)))
@pytest.mark.parametrize("hash_fn", [sha256, sha256_cbor])
@@ -1043,7 +1091,7 @@ def test_request_block_hasher_incremental_append_with_multiple_mm_features(hash_
)
expected_second_hash = hash_fn(
(incremental.block_hashes[0], (4, 5, 6, 7), (("A", 0), ("B", 2)))
(incremental.block_hashes[0], (4, 5, 6, 7), (("mm", "A", 0), ("mm", "B", 2)))
)
assert incremental.block_hashes[1] == expected_second_hash
assert incremental.block_hashes == fresh.block_hashes
@@ -3657,7 +3705,7 @@ def test_request_block_hasher_with_prompt_embeds(hash_fn: Callable[[Any], bytes]
(
kv_cache_utils.NONE_HASH,
tuple(prompt_token_ids[:block_size]),
(block1_embeds_hash,),
(("prompt_embeds", block1_embeds_hash),),
)
)
assert block_hashes[0] == expected_hash1
@@ -3669,7 +3717,7 @@ def test_request_block_hasher_with_prompt_embeds(hash_fn: Callable[[Any], bytes]
(
block_hashes[0],
tuple(prompt_token_ids[block_size:num_tokens]),
(block2_embeds_hash,),
(("prompt_embeds", block2_embeds_hash),),
)
)
assert block_hashes[1] == expected_hash2
@@ -3706,7 +3754,7 @@ def test_request_with_prompt_embeds_and_mm_inputs(hash_fn: Callable[[Any], bytes
(
kv_cache_utils.NONE_HASH,
tuple(prompt_token_ids[:block_size]),
(("hash1", 0), block1_embeds_hash),
(("mm", "hash1", 0), ("prompt_embeds", block1_embeds_hash)),
)
)
assert block_hashes[0] == expected_hash1
@@ -3718,7 +3766,7 @@ def test_request_with_prompt_embeds_and_mm_inputs(hash_fn: Callable[[Any], bytes
(
block_hashes[0],
tuple(prompt_token_ids[block_size:num_tokens]),
(("hash2", 0), block2_embeds_hash),
(("mm", "hash2", 0), ("prompt_embeds", block2_embeds_hash)),
)
)
assert block_hashes[1] == expected_hash2
+14 -6
View File
@@ -2901,21 +2901,21 @@ def test_mm_prefix_caching():
(
kv_cache_utils.NONE_HASH,
tuple(all_token_ids[:block_size]),
(("aaa", 11),),
(("mm", "aaa", 11),),
)
)
assert block_hashes[1] == sha256(
(
block_hashes[0],
tuple(all_token_ids[block_size : block_size * 2]),
(("aaa", -5), ("bbb", 14)),
(("mm", "aaa", -5), ("mm", "bbb", 14)),
)
)
assert block_hashes[2] == sha256(
(
block_hashes[1],
tuple(all_token_ids[block_size * 2 : block_size * 3]),
(("bbb", -2),),
(("mm", "bbb", -2),),
)
)
@@ -2938,7 +2938,7 @@ def test_mm_prefix_caching():
(
block_hashes[2],
tuple(all_token_ids[3 * block_size :] + [8] * 5),
(("ccc", 0),),
(("mm", "ccc", 0),),
)
)
@@ -2984,7 +2984,11 @@ def test_cache_key_salting():
block_hashes = req0.block_hashes
assert len(block_hashes) == 3
assert block_hashes[0] == sha256(
(kv_cache_utils.NONE_HASH, tuple(token_ids[:block_size]), ("salt1",))
(
kv_cache_utils.NONE_HASH,
tuple(token_ids[:block_size]),
(("cache_salt", "salt1"),),
)
)
assert block_hashes[1] == sha256(
(block_hashes[0], tuple(token_ids[block_size : block_size * 2]), None)
@@ -3029,7 +3033,11 @@ def test_cache_key_salting():
block_hashes = req2.block_hashes
assert len(block_hashes) == 3
assert block_hashes[0] == sha256(
(kv_cache_utils.NONE_HASH, tuple(token_ids[:block_size]), ("salt2",))
(
kv_cache_utils.NONE_HASH,
tuple(token_ids[:block_size]),
(("cache_salt", "salt2"),),
)
)
assert block_hashes[1] == sha256(
(block_hashes[0], tuple(token_ids[block_size : block_size * 2]), None)
+5 -3
View File
@@ -68,9 +68,11 @@ class BlockStored(KVCacheEvent):
lora_name: str | None
"""Extra keys used in block hash computation, one entry per block in
block_hashes. Each entry contains MM identifiers, LoRA name, cache_salt,
prompt embedding hashes, etc. for that specific block. Exposed for external
KV cache consumers to reconstruct block hashes.
block_hashes. Each entry contains, in order, the LoRA name (if
`lora_name` is set), `(mm_identifier, offset)` pairs, the cache_salt
string and prompt embedding digests for that specific block. These are
published untagged; the block hash itself uses each value tagged with
its source, e.g. `("lora", name)` or `("mm", mm_identifier, offset)`.
"""
extra_keys: list[tuple[Any, ...] | None] | None = None
@@ -31,6 +31,7 @@ from vllm.v1.core.kv_cache_utils import (
BlockHash,
maybe_convert_block_hash,
resolve_block_hashes,
to_event_extra_keys,
)
from vllm.v1.kv_cache_interface import (
KVCacheGroupSpec,
@@ -393,9 +394,7 @@ class OffloadingEventsTracker:
lora_id=meta.lora_id,
medium=_MEDIUM_TO_EVENT_STR[event.medium],
lora_name=meta.lora_name,
extra_keys=(
list(meta.extra_keys) if meta.extra_keys is not None else None
),
extra_keys=to_event_extra_keys(meta.extra_keys),
group_idx=meta.group_idx,
kv_cache_spec_kind=meta.kv_cache_spec.kv_cache_spec_kind,
kv_cache_spec_sliding_window=(
+3 -2
View File
@@ -25,6 +25,7 @@ from vllm.v1.core.kv_cache_utils import (
make_block_hash_with_group_id,
maybe_convert_block_hash,
resolve_block_hashes,
to_event_extra_keys,
)
from vllm.v1.request import Request
@@ -367,7 +368,7 @@ class BlockPool:
lora_id=request.lora_request.adapter_id if request.lora_request else None,
medium=self.medium,
lora_name=request.lora_request.name if request.lora_request else None,
extra_keys=extra_keys_list if extra_keys_list else None,
extra_keys=to_event_extra_keys(extra_keys_list),
group_idx=kv_cache_group_id,
session_id=request.session_id,
)
@@ -557,7 +558,7 @@ class BlockPool:
lora_name=request.lora_request.name
if request.lora_request
else None,
extra_keys=[extra_keys],
extra_keys=to_event_extra_keys([extra_keys]),
group_idx=kv_cache_group_id,
session_id=request.session_id,
)
+41 -10
View File
@@ -502,8 +502,8 @@ def _gen_mm_extra_hash_keys(
) -> tuple[list[Any], int]:
"""Generate extra keys related to MultiModal request for block hash
computation. For multi-modal inputs, the extra keys are
(mm_hash, start_offset) that indicate a mm input contained in the
block and its starting offset in the block tokens.
("mm", mm_hash, start_offset) tuples that indicate a mm input contained
in the block and its starting offset in the block tokens.
Args:
request: The request object.
@@ -549,7 +549,7 @@ def _gen_mm_extra_hash_keys(
# relative to the start of the block so prefix-cache keys stay
# distinct when the same MM item appears at different positions
# within otherwise-identical placeholder blocks.
extra_keys.append((mm_feature.identifier, offset - start_token_idx))
extra_keys.append(("mm", mm_feature.identifier, offset - start_token_idx))
if end_token_idx >= offset + length:
# If this block contains the end of the current mm input,
@@ -565,7 +565,7 @@ def _gen_mm_extra_hash_keys(
return extra_keys, curr_mm_idx
def _gen_lora_extra_hash_keys(request: Request) -> list[str]:
def _gen_lora_extra_hash_keys(request: Request) -> list[tuple[str, str]]:
"""Generate extra keys related to LoRA for block hash computation.
Args:
@@ -578,12 +578,12 @@ def _gen_lora_extra_hash_keys(request: Request) -> list[str]:
"""
if not request.lora_request:
return []
return [request.lora_request.lora_name]
return [("lora", request.lora_request.lora_name)]
def _gen_prompt_embeds_extra_hash_keys(
request: Request, start_token_idx: int, end_token_idx: int
) -> list[bytes]:
) -> list[tuple[str, bytes]]:
"""Generate extra keys related to prompt embeds for block hash computation.
Args:
@@ -605,7 +605,7 @@ def _gen_prompt_embeds_extra_hash_keys(
# Hash prompt embeds once per block and cache on request
embeds_hash = hashlib.sha256(tensor_data(block_prompt_embeds)).digest()
request._prompt_embeds_per_block_hashes[block_range] = embeds_hash
return [embeds_hash]
return [("prompt_embeds", embeds_hash)]
def generate_block_hash_extra_keys(
@@ -629,9 +629,11 @@ def generate_block_hash_extra_keys(
mm_extra_keys, new_start_mm_idx = _gen_mm_extra_hash_keys(
request, start_token_idx, end_token_idx, start_mm_idx
)
lora_extra_keys: list[str] = _gen_lora_extra_hash_keys(request)
cache_salt_keys: list[str] = (
[request.cache_salt] if (start_token_idx == 0 and request.cache_salt) else []
lora_extra_keys = _gen_lora_extra_hash_keys(request)
cache_salt_keys: list[tuple[str, str]] = (
[("cache_salt", request.cache_salt)]
if (start_token_idx == 0 and request.cache_salt)
else []
)
prompt_embeds_keys = _gen_prompt_embeds_extra_hash_keys(
request, start_token_idx, end_token_idx
@@ -647,6 +649,35 @@ def generate_block_hash_extra_keys(
return tuple(extra_keys), new_start_mm_idx
def to_event_extra_keys(
extra_keys: Iterable[tuple[Any, ...] | None] | None,
) -> list[tuple[Any, ...] | None] | None:
"""Convert block-hash extra keys to the untagged per-block list published
in KV events.
External KV event consumers parse the pre-tagging shapes: bare LoRA names
and cache salts, ``(mm_identifier, offset)`` pairs and bare prompt-embeds
digests. Events keep that format until consumers handle the tagged keys.
Args:
extra_keys: One entry per block, each as returned by
`generate_block_hash_extra_keys`, or None.
Returns:
One untagged entry per block, or None if there are no entries.
"""
if not extra_keys:
return None
event_keys = [
None
if keys is None
else tuple(key[1:] if key[0] == "mm" else key[1] for key in keys)
for keys in extra_keys
]
return event_keys or None
def hash_block_tokens(
hash_function: Callable[[Any], bytes],
parent_block_hash: BlockHash | None,
+2 -3
View File
@@ -34,6 +34,7 @@ from vllm.v1.core.kv_cache_utils import (
maybe_convert_block_hash,
resolve_block_hashes,
resolve_dcp_kv_block_size,
to_event_extra_keys,
)
from vllm.v1.core.sched.output import SchedulerOutput
from vllm.v1.kv_cache_interface import (
@@ -1147,9 +1148,7 @@ class SimpleCPUOffloadScheduler:
lora_id=meta.lora_id if meta else None,
medium=self.kv_event_medium,
lora_name=meta.lora_name if meta else None,
extra_keys=(
[extra_keys] if extra_keys is not None else None
),
extra_keys=to_event_extra_keys(extra_keys and [extra_keys]),
group_idx=group_idx,
kv_cache_spec_kind=get_kv_cache_spec_kind(spec).value,
kv_cache_spec_sliding_window=(