[Core][BugFix] Tag prefix-cache extra keys by source (#51899)
Signed-off-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Lucas Bourtoule <35483370+dhalf@users.noreply.github.com> Co-authored-by: Tai An <antai12232931@outlook.com> Co-authored-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Lucas Bourtoule
Tai An
Nick Hill
Claude Opus 5.5
parent
09c47db1ca
commit
765872e7ed
@@ -54,6 +54,7 @@ from vllm.v1.core.kv_cache_utils import (
|
||||
is_kv_cache_spec_uniform,
|
||||
make_block_hash_with_group_id,
|
||||
tensor_data,
|
||||
to_event_extra_keys,
|
||||
)
|
||||
from vllm.v1.hisparse.layout import (
|
||||
create_hisparse_layout,
|
||||
@@ -822,12 +823,12 @@ def test_generate_block_hash_extra_keys():
|
||||
|
||||
# Test with no extra keys
|
||||
extra_keys, next_mm_idx = generate_block_hash_extra_keys(request, 0, 5, 0)
|
||||
assert extra_keys == (("hash1", 0),)
|
||||
assert extra_keys == (("mm", "hash1", 0),)
|
||||
assert next_mm_idx == 1
|
||||
|
||||
# Test with partial overlap
|
||||
extra_keys, next_mm_idx = generate_block_hash_extra_keys(request, 3, 8, 0)
|
||||
assert extra_keys == (("hash1", -3),)
|
||||
assert extra_keys == (("mm", "hash1", -3),)
|
||||
assert next_mm_idx == 1
|
||||
|
||||
# Test with no overlap
|
||||
@@ -837,7 +838,7 @@ def test_generate_block_hash_extra_keys():
|
||||
|
||||
# Test with multiple extra keys
|
||||
extra_keys, next_mm_idx = generate_block_hash_extra_keys(request, 0, 15, 0)
|
||||
assert extra_keys == (("hash1", 0), ("hash2", 10))
|
||||
assert extra_keys == (("mm", "hash1", 0), ("mm", "hash2", 10))
|
||||
assert next_mm_idx == 2
|
||||
|
||||
|
||||
@@ -865,9 +866,9 @@ def test_generate_block_hash_extra_keys_cache_salt():
|
||||
|
||||
# salt is added for the first token
|
||||
extra_keys, _ = generate_block_hash_extra_keys(request, 0, 1, 0)
|
||||
assert extra_keys == ("salt",)
|
||||
assert extra_keys == (("cache_salt", "salt"),)
|
||||
extra_keys, _ = generate_block_hash_extra_keys(request, 0, 10, 0)
|
||||
assert extra_keys == ("salt",)
|
||||
assert extra_keys == (("cache_salt", "salt"),)
|
||||
|
||||
# no salt added for other tokens
|
||||
extra_keys, _ = generate_block_hash_extra_keys(request, 1, 2, 0)
|
||||
@@ -888,7 +889,7 @@ def test_generate_block_hash_extra_keys_cache_salt():
|
||||
|
||||
# Test with no extra keys
|
||||
extra_keys, next_mm_idx = generate_block_hash_extra_keys(request_mm, 0, 5, 0)
|
||||
assert extra_keys == (("hash1", 0), "salt")
|
||||
assert extra_keys == (("mm", "hash1", 0), ("cache_salt", "salt"))
|
||||
assert next_mm_idx == 1
|
||||
|
||||
|
||||
@@ -906,13 +907,13 @@ def test_generate_block_hash_extra_keys_prompt_embeds():
|
||||
extra_keys, _ = generate_block_hash_extra_keys(request, 0, 5, 0)
|
||||
expected_embeds = prompt_embeds[0:5]
|
||||
expected_hash = hashlib.sha256(kv_cache_utils.tensor_data(expected_embeds)).digest()
|
||||
assert extra_keys == (expected_hash,)
|
||||
assert extra_keys == (("prompt_embeds", expected_hash),)
|
||||
|
||||
# Test with prompt embeds for the second block
|
||||
extra_keys, _ = generate_block_hash_extra_keys(request, 5, 10, 0)
|
||||
expected_embeds = prompt_embeds[5:10]
|
||||
expected_hash = hashlib.sha256(kv_cache_utils.tensor_data(expected_embeds)).digest()
|
||||
assert extra_keys == (expected_hash,)
|
||||
assert extra_keys == (("prompt_embeds", expected_hash),)
|
||||
|
||||
|
||||
def test_generate_block_hash_extra_keys_prompt_embeds_cached(monkeypatch):
|
||||
@@ -976,13 +977,58 @@ def test_generate_block_hash_extra_keys_lora():
|
||||
)
|
||||
|
||||
extra_keys, _ = generate_block_hash_extra_keys(request, 0, 3, 0)
|
||||
assert extra_keys == ("test_lora_adapter",)
|
||||
assert extra_keys == (("lora", "test_lora_adapter"),)
|
||||
|
||||
request.lora_request = None
|
||||
extra_keys, _ = generate_block_hash_extra_keys(request, 0, 3, 0)
|
||||
assert extra_keys is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize("hash_fn", [sha256, sha256_cbor])
|
||||
def test_lora_name_and_cache_salt_block_hashes_do_not_collide(hash_fn):
|
||||
"""A cache_salt equal to a LoRA name must not share that LoRA's blocks."""
|
||||
lora_request = LoRARequest(
|
||||
lora_name="foo", lora_int_id=1, lora_path="/path/to/lora"
|
||||
)
|
||||
lora_req = Request(
|
||||
request_id="lora",
|
||||
prompt_token_ids=[0, 1, 2],
|
||||
sampling_params=SamplingParams(max_tokens=1),
|
||||
pooling_params=None,
|
||||
lora_request=lora_request,
|
||||
block_hasher=get_request_block_hasher(3, hash_fn),
|
||||
)
|
||||
salted_req = make_request("salted", [0, 1, 2], hash_fn=hash_fn, cache_salt="foo")
|
||||
|
||||
assert lora_req.block_hashes[0] != salted_req.block_hashes[0]
|
||||
|
||||
|
||||
def test_to_event_extra_keys_keeps_untagged_event_format():
|
||||
"""KV events keep publishing the extra-key shapes consumers already parse."""
|
||||
request = make_request(
|
||||
"0",
|
||||
list(range(10)),
|
||||
mm_positions=[PlaceholderRange(offset=2, length=3)],
|
||||
mm_hashes=["hash1"],
|
||||
cache_salt="salt",
|
||||
)
|
||||
request.lora_request = LoRARequest(
|
||||
lora_name="adapter", lora_int_id=1, lora_path="/path/to/lora"
|
||||
)
|
||||
|
||||
extra_keys, _ = generate_block_hash_extra_keys(request, 0, 5, 0)
|
||||
|
||||
assert extra_keys == (
|
||||
("lora", "adapter"),
|
||||
("mm", "hash1", 2),
|
||||
("cache_salt", "salt"),
|
||||
)
|
||||
event_keys = ("adapter", ("hash1", 2), "salt")
|
||||
assert to_event_extra_keys([extra_keys, None]) == [event_keys, None]
|
||||
assert to_event_extra_keys([]) is None
|
||||
assert to_event_extra_keys(None) is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize("hash_fn", [sha256, sha256_cbor])
|
||||
def test_hash_block_tokens(hash_fn):
|
||||
parent_block_hash = BlockHash(b"123")
|
||||
@@ -1013,9 +1059,11 @@ def test_request_block_hasher(hash_fn):
|
||||
block_hashes = request.block_hashes
|
||||
assert len(block_hashes) == 2
|
||||
assert block_hashes[0] == hash_fn(
|
||||
(kv_cache_utils.NONE_HASH, (0, 1, 2), (("hash1", 0),))
|
||||
(kv_cache_utils.NONE_HASH, (0, 1, 2), (("mm", "hash1", 0),))
|
||||
)
|
||||
assert block_hashes[1] == hash_fn(
|
||||
(block_hashes[0], (3, 4, 5), (("mm", "hash2", 0),))
|
||||
)
|
||||
assert block_hashes[1] == hash_fn((block_hashes[0], (3, 4, 5), (("hash2", 0),)))
|
||||
|
||||
|
||||
@pytest.mark.parametrize("hash_fn", [sha256, sha256_cbor])
|
||||
@@ -1043,7 +1091,7 @@ def test_request_block_hasher_incremental_append_with_multiple_mm_features(hash_
|
||||
)
|
||||
|
||||
expected_second_hash = hash_fn(
|
||||
(incremental.block_hashes[0], (4, 5, 6, 7), (("A", 0), ("B", 2)))
|
||||
(incremental.block_hashes[0], (4, 5, 6, 7), (("mm", "A", 0), ("mm", "B", 2)))
|
||||
)
|
||||
assert incremental.block_hashes[1] == expected_second_hash
|
||||
assert incremental.block_hashes == fresh.block_hashes
|
||||
@@ -3657,7 +3705,7 @@ def test_request_block_hasher_with_prompt_embeds(hash_fn: Callable[[Any], bytes]
|
||||
(
|
||||
kv_cache_utils.NONE_HASH,
|
||||
tuple(prompt_token_ids[:block_size]),
|
||||
(block1_embeds_hash,),
|
||||
(("prompt_embeds", block1_embeds_hash),),
|
||||
)
|
||||
)
|
||||
assert block_hashes[0] == expected_hash1
|
||||
@@ -3669,7 +3717,7 @@ def test_request_block_hasher_with_prompt_embeds(hash_fn: Callable[[Any], bytes]
|
||||
(
|
||||
block_hashes[0],
|
||||
tuple(prompt_token_ids[block_size:num_tokens]),
|
||||
(block2_embeds_hash,),
|
||||
(("prompt_embeds", block2_embeds_hash),),
|
||||
)
|
||||
)
|
||||
assert block_hashes[1] == expected_hash2
|
||||
@@ -3706,7 +3754,7 @@ def test_request_with_prompt_embeds_and_mm_inputs(hash_fn: Callable[[Any], bytes
|
||||
(
|
||||
kv_cache_utils.NONE_HASH,
|
||||
tuple(prompt_token_ids[:block_size]),
|
||||
(("hash1", 0), block1_embeds_hash),
|
||||
(("mm", "hash1", 0), ("prompt_embeds", block1_embeds_hash)),
|
||||
)
|
||||
)
|
||||
assert block_hashes[0] == expected_hash1
|
||||
@@ -3718,7 +3766,7 @@ def test_request_with_prompt_embeds_and_mm_inputs(hash_fn: Callable[[Any], bytes
|
||||
(
|
||||
block_hashes[0],
|
||||
tuple(prompt_token_ids[block_size:num_tokens]),
|
||||
(("hash2", 0), block2_embeds_hash),
|
||||
(("mm", "hash2", 0), ("prompt_embeds", block2_embeds_hash)),
|
||||
)
|
||||
)
|
||||
assert block_hashes[1] == expected_hash2
|
||||
|
||||
@@ -2901,21 +2901,21 @@ def test_mm_prefix_caching():
|
||||
(
|
||||
kv_cache_utils.NONE_HASH,
|
||||
tuple(all_token_ids[:block_size]),
|
||||
(("aaa", 11),),
|
||||
(("mm", "aaa", 11),),
|
||||
)
|
||||
)
|
||||
assert block_hashes[1] == sha256(
|
||||
(
|
||||
block_hashes[0],
|
||||
tuple(all_token_ids[block_size : block_size * 2]),
|
||||
(("aaa", -5), ("bbb", 14)),
|
||||
(("mm", "aaa", -5), ("mm", "bbb", 14)),
|
||||
)
|
||||
)
|
||||
assert block_hashes[2] == sha256(
|
||||
(
|
||||
block_hashes[1],
|
||||
tuple(all_token_ids[block_size * 2 : block_size * 3]),
|
||||
(("bbb", -2),),
|
||||
(("mm", "bbb", -2),),
|
||||
)
|
||||
)
|
||||
|
||||
@@ -2938,7 +2938,7 @@ def test_mm_prefix_caching():
|
||||
(
|
||||
block_hashes[2],
|
||||
tuple(all_token_ids[3 * block_size :] + [8] * 5),
|
||||
(("ccc", 0),),
|
||||
(("mm", "ccc", 0),),
|
||||
)
|
||||
)
|
||||
|
||||
@@ -2984,7 +2984,11 @@ def test_cache_key_salting():
|
||||
block_hashes = req0.block_hashes
|
||||
assert len(block_hashes) == 3
|
||||
assert block_hashes[0] == sha256(
|
||||
(kv_cache_utils.NONE_HASH, tuple(token_ids[:block_size]), ("salt1",))
|
||||
(
|
||||
kv_cache_utils.NONE_HASH,
|
||||
tuple(token_ids[:block_size]),
|
||||
(("cache_salt", "salt1"),),
|
||||
)
|
||||
)
|
||||
assert block_hashes[1] == sha256(
|
||||
(block_hashes[0], tuple(token_ids[block_size : block_size * 2]), None)
|
||||
@@ -3029,7 +3033,11 @@ def test_cache_key_salting():
|
||||
block_hashes = req2.block_hashes
|
||||
assert len(block_hashes) == 3
|
||||
assert block_hashes[0] == sha256(
|
||||
(kv_cache_utils.NONE_HASH, tuple(token_ids[:block_size]), ("salt2",))
|
||||
(
|
||||
kv_cache_utils.NONE_HASH,
|
||||
tuple(token_ids[:block_size]),
|
||||
(("cache_salt", "salt2"),),
|
||||
)
|
||||
)
|
||||
assert block_hashes[1] == sha256(
|
||||
(block_hashes[0], tuple(token_ids[block_size : block_size * 2]), None)
|
||||
|
||||
@@ -68,9 +68,11 @@ class BlockStored(KVCacheEvent):
|
||||
lora_name: str | None
|
||||
|
||||
"""Extra keys used in block hash computation, one entry per block in
|
||||
block_hashes. Each entry contains MM identifiers, LoRA name, cache_salt,
|
||||
prompt embedding hashes, etc. for that specific block. Exposed for external
|
||||
KV cache consumers to reconstruct block hashes.
|
||||
block_hashes. Each entry contains, in order, the LoRA name (if
|
||||
`lora_name` is set), `(mm_identifier, offset)` pairs, the cache_salt
|
||||
string and prompt embedding digests for that specific block. These are
|
||||
published untagged; the block hash itself uses each value tagged with
|
||||
its source, e.g. `("lora", name)` or `("mm", mm_identifier, offset)`.
|
||||
"""
|
||||
extra_keys: list[tuple[Any, ...] | None] | None = None
|
||||
|
||||
|
||||
@@ -31,6 +31,7 @@ from vllm.v1.core.kv_cache_utils import (
|
||||
BlockHash,
|
||||
maybe_convert_block_hash,
|
||||
resolve_block_hashes,
|
||||
to_event_extra_keys,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
KVCacheGroupSpec,
|
||||
@@ -393,9 +394,7 @@ class OffloadingEventsTracker:
|
||||
lora_id=meta.lora_id,
|
||||
medium=_MEDIUM_TO_EVENT_STR[event.medium],
|
||||
lora_name=meta.lora_name,
|
||||
extra_keys=(
|
||||
list(meta.extra_keys) if meta.extra_keys is not None else None
|
||||
),
|
||||
extra_keys=to_event_extra_keys(meta.extra_keys),
|
||||
group_idx=meta.group_idx,
|
||||
kv_cache_spec_kind=meta.kv_cache_spec.kv_cache_spec_kind,
|
||||
kv_cache_spec_sliding_window=(
|
||||
|
||||
@@ -25,6 +25,7 @@ from vllm.v1.core.kv_cache_utils import (
|
||||
make_block_hash_with_group_id,
|
||||
maybe_convert_block_hash,
|
||||
resolve_block_hashes,
|
||||
to_event_extra_keys,
|
||||
)
|
||||
from vllm.v1.request import Request
|
||||
|
||||
@@ -367,7 +368,7 @@ class BlockPool:
|
||||
lora_id=request.lora_request.adapter_id if request.lora_request else None,
|
||||
medium=self.medium,
|
||||
lora_name=request.lora_request.name if request.lora_request else None,
|
||||
extra_keys=extra_keys_list if extra_keys_list else None,
|
||||
extra_keys=to_event_extra_keys(extra_keys_list),
|
||||
group_idx=kv_cache_group_id,
|
||||
session_id=request.session_id,
|
||||
)
|
||||
@@ -557,7 +558,7 @@ class BlockPool:
|
||||
lora_name=request.lora_request.name
|
||||
if request.lora_request
|
||||
else None,
|
||||
extra_keys=[extra_keys],
|
||||
extra_keys=to_event_extra_keys([extra_keys]),
|
||||
group_idx=kv_cache_group_id,
|
||||
session_id=request.session_id,
|
||||
)
|
||||
|
||||
@@ -502,8 +502,8 @@ def _gen_mm_extra_hash_keys(
|
||||
) -> tuple[list[Any], int]:
|
||||
"""Generate extra keys related to MultiModal request for block hash
|
||||
computation. For multi-modal inputs, the extra keys are
|
||||
(mm_hash, start_offset) that indicate a mm input contained in the
|
||||
block and its starting offset in the block tokens.
|
||||
("mm", mm_hash, start_offset) tuples that indicate a mm input contained
|
||||
in the block and its starting offset in the block tokens.
|
||||
|
||||
Args:
|
||||
request: The request object.
|
||||
@@ -549,7 +549,7 @@ def _gen_mm_extra_hash_keys(
|
||||
# relative to the start of the block so prefix-cache keys stay
|
||||
# distinct when the same MM item appears at different positions
|
||||
# within otherwise-identical placeholder blocks.
|
||||
extra_keys.append((mm_feature.identifier, offset - start_token_idx))
|
||||
extra_keys.append(("mm", mm_feature.identifier, offset - start_token_idx))
|
||||
|
||||
if end_token_idx >= offset + length:
|
||||
# If this block contains the end of the current mm input,
|
||||
@@ -565,7 +565,7 @@ def _gen_mm_extra_hash_keys(
|
||||
return extra_keys, curr_mm_idx
|
||||
|
||||
|
||||
def _gen_lora_extra_hash_keys(request: Request) -> list[str]:
|
||||
def _gen_lora_extra_hash_keys(request: Request) -> list[tuple[str, str]]:
|
||||
"""Generate extra keys related to LoRA for block hash computation.
|
||||
|
||||
Args:
|
||||
@@ -578,12 +578,12 @@ def _gen_lora_extra_hash_keys(request: Request) -> list[str]:
|
||||
"""
|
||||
if not request.lora_request:
|
||||
return []
|
||||
return [request.lora_request.lora_name]
|
||||
return [("lora", request.lora_request.lora_name)]
|
||||
|
||||
|
||||
def _gen_prompt_embeds_extra_hash_keys(
|
||||
request: Request, start_token_idx: int, end_token_idx: int
|
||||
) -> list[bytes]:
|
||||
) -> list[tuple[str, bytes]]:
|
||||
"""Generate extra keys related to prompt embeds for block hash computation.
|
||||
|
||||
Args:
|
||||
@@ -605,7 +605,7 @@ def _gen_prompt_embeds_extra_hash_keys(
|
||||
# Hash prompt embeds once per block and cache on request
|
||||
embeds_hash = hashlib.sha256(tensor_data(block_prompt_embeds)).digest()
|
||||
request._prompt_embeds_per_block_hashes[block_range] = embeds_hash
|
||||
return [embeds_hash]
|
||||
return [("prompt_embeds", embeds_hash)]
|
||||
|
||||
|
||||
def generate_block_hash_extra_keys(
|
||||
@@ -629,9 +629,11 @@ def generate_block_hash_extra_keys(
|
||||
mm_extra_keys, new_start_mm_idx = _gen_mm_extra_hash_keys(
|
||||
request, start_token_idx, end_token_idx, start_mm_idx
|
||||
)
|
||||
lora_extra_keys: list[str] = _gen_lora_extra_hash_keys(request)
|
||||
cache_salt_keys: list[str] = (
|
||||
[request.cache_salt] if (start_token_idx == 0 and request.cache_salt) else []
|
||||
lora_extra_keys = _gen_lora_extra_hash_keys(request)
|
||||
cache_salt_keys: list[tuple[str, str]] = (
|
||||
[("cache_salt", request.cache_salt)]
|
||||
if (start_token_idx == 0 and request.cache_salt)
|
||||
else []
|
||||
)
|
||||
prompt_embeds_keys = _gen_prompt_embeds_extra_hash_keys(
|
||||
request, start_token_idx, end_token_idx
|
||||
@@ -647,6 +649,35 @@ def generate_block_hash_extra_keys(
|
||||
return tuple(extra_keys), new_start_mm_idx
|
||||
|
||||
|
||||
def to_event_extra_keys(
|
||||
extra_keys: Iterable[tuple[Any, ...] | None] | None,
|
||||
) -> list[tuple[Any, ...] | None] | None:
|
||||
"""Convert block-hash extra keys to the untagged per-block list published
|
||||
in KV events.
|
||||
|
||||
External KV event consumers parse the pre-tagging shapes: bare LoRA names
|
||||
and cache salts, ``(mm_identifier, offset)`` pairs and bare prompt-embeds
|
||||
digests. Events keep that format until consumers handle the tagged keys.
|
||||
|
||||
Args:
|
||||
extra_keys: One entry per block, each as returned by
|
||||
`generate_block_hash_extra_keys`, or None.
|
||||
|
||||
Returns:
|
||||
One untagged entry per block, or None if there are no entries.
|
||||
|
||||
"""
|
||||
if not extra_keys:
|
||||
return None
|
||||
event_keys = [
|
||||
None
|
||||
if keys is None
|
||||
else tuple(key[1:] if key[0] == "mm" else key[1] for key in keys)
|
||||
for keys in extra_keys
|
||||
]
|
||||
return event_keys or None
|
||||
|
||||
|
||||
def hash_block_tokens(
|
||||
hash_function: Callable[[Any], bytes],
|
||||
parent_block_hash: BlockHash | None,
|
||||
|
||||
@@ -34,6 +34,7 @@ from vllm.v1.core.kv_cache_utils import (
|
||||
maybe_convert_block_hash,
|
||||
resolve_block_hashes,
|
||||
resolve_dcp_kv_block_size,
|
||||
to_event_extra_keys,
|
||||
)
|
||||
from vllm.v1.core.sched.output import SchedulerOutput
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
@@ -1147,9 +1148,7 @@ class SimpleCPUOffloadScheduler:
|
||||
lora_id=meta.lora_id if meta else None,
|
||||
medium=self.kv_event_medium,
|
||||
lora_name=meta.lora_name if meta else None,
|
||||
extra_keys=(
|
||||
[extra_keys] if extra_keys is not None else None
|
||||
),
|
||||
extra_keys=to_event_extra_keys(extra_keys and [extra_keys]),
|
||||
group_idx=group_idx,
|
||||
kv_cache_spec_kind=get_kv_cache_spec_kind(spec).value,
|
||||
kv_cache_spec_sliding_window=(
|
||||
|
||||
Reference in New Issue
Block a user