[feat] Add per-block extra_keys to KV events (#33304)

Signed-off-by: zhongdaor-nv <zhongdaor@nvidia.com>
Co-authored-by: Nicolò Lucchesi <nlucches@redhat.com>
Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
This commit is contained in:
zhongdaor-nv
2026-02-20 20:11:40 -08:00
committed by GitHub
co-authored by Nicolò Lucchesi Cyrus Leung
parent 991d6bff38
commit a0fe7ea2f0
6 changed files with 100 additions and 21 deletions
+8
View File
@@ -60,6 +60,13 @@ class BlockStored(KVCacheEvent):
medium: str | None
lora_name: str | None
extra_keys: list[tuple[Any, ...] | None] | None = None
"""Extra keys used in block hash computation, one entry per block in
block_hashes. Each entry contains MM identifiers, LoRA name, cache_salt,
prompt embedding hashes, etc. for that specific block. Exposed for external
KV cache consumers to reconstruct block hashes.
"""
def __hash__(self) -> int:
return hash(
(
@@ -69,6 +76,7 @@ class BlockStored(KVCacheEvent):
self.block_size,
self.lora_id,
self.medium,
tuple(self.extra_keys) if self.extra_keys else None,
)
)
+23 -3
View File
@@ -20,6 +20,7 @@ from vllm.v1.core.kv_cache_utils import (
ExternalBlockHash,
FreeKVCacheBlockQueue,
KVCacheBlock,
generate_block_hash_extra_keys,
get_block_hash,
make_block_hash_with_group_id,
maybe_convert_block_hash,
@@ -279,13 +280,31 @@ class BlockPool:
block_hashes[num_cached_blocks - 1]
)
# Calculate token range for the blocks being cached
start_token_idx = num_cached_blocks * block_size
end_token_idx = num_full_blocks * block_size
# Generate extra keys for each block individually.
# Each block may have different extra_keys (e.g., different MM
# features, or cache_salt only for the first block).
# Skip null blocks to match the length of new_hashes.
extra_keys_list: list[tuple[Any, ...] | None] = []
curr_mm_idx = 0
for i in range(num_cached_blocks, num_full_blocks):
if blocks[i].is_null:
continue
block_start = i * block_size
block_end = block_start + block_size
extra_keys, curr_mm_idx = generate_block_hash_extra_keys(
request, block_start, block_end, curr_mm_idx
)
extra_keys_list.append(extra_keys)
self.kv_event_queue.append(
BlockStored(
block_hashes=new_hashes,
parent_block_hash=parent_block_hash,
token_ids=request.all_token_ids[
num_cached_blocks * block_size : num_full_blocks * block_size
],
token_ids=request.all_token_ids[start_token_idx:end_token_idx],
block_size=block_size,
lora_id=request.lora_request.adapter_id
if request.lora_request
@@ -294,6 +313,7 @@ class BlockPool:
lora_name=request.lora_request.name
if request.lora_request
else None,
extra_keys=extra_keys_list if extra_keys_list else None,
)
)
+12 -6
View File
@@ -3,6 +3,7 @@
"""KV-Cache Utilities."""
import copy
import hashlib
import os
from collections import defaultdict
from collections.abc import Callable, Iterable, Iterator, Sequence
@@ -475,14 +476,19 @@ def _gen_prompt_embeds_extra_hash_keys(
end_token_idx: The end token index of the block.
Returns:
Return prompt embeddings data of the request if it has prompt embeds.
Return empty list otherwise.
Return a stable hash of the block prompt embeddings if prompt embeds
are present. Return empty list otherwise.
"""
if request.prompt_embeds is None:
return []
block_prompt_embeds = request.prompt_embeds[start_token_idx:end_token_idx]
embeds_bytes = tensor_data(block_prompt_embeds).tobytes()
return [embeds_bytes]
block_range = (start_token_idx, end_token_idx)
embeds_hash = request._prompt_embeds_per_block_hashes.get(block_range)
if embeds_hash is None:
block_prompt_embeds = request.prompt_embeds[start_token_idx:end_token_idx]
# Hash prompt embeds once per block and cache on request
embeds_hash = hashlib.sha256(tensor_data(block_prompt_embeds)).digest()
request._prompt_embeds_per_block_hashes[block_range] = embeds_hash
return [embeds_hash]
def generate_block_hash_extra_keys(
@@ -490,7 +496,7 @@ def generate_block_hash_extra_keys(
) -> tuple[tuple[Any, ...] | None, int]:
"""Generate extra keys for the block hash. The extra keys can come from
the multi-modal inputs, request specific metadata (e.g., LoRA names), and
data from prompt embeddings.
hashed data from prompt embeddings.
Args:
request: The request object.
+3
View File
@@ -114,6 +114,9 @@ class Request:
self.prompt_token_ids = prompt_token_ids
self.prompt_embeds = prompt_embeds
# Cache per-block prompt-embed hashes to avoid rehashing the same
# tensor slices when generating extra keys.
self._prompt_embeds_per_block_hashes: dict[tuple[int, int], bytes] = {}
self.num_prompt_tokens = length_from_prompt_token_ids_or_embeds(
prompt_token_ids, prompt_embeds
)