[feat] Add per-block extra_keys to KV events (#33304)
Signed-off-by: zhongdaor-nv <zhongdaor@nvidia.com> Co-authored-by: Nicolò Lucchesi <nlucches@redhat.com> Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
This commit is contained in:
co-authored by
Nicolò Lucchesi
Cyrus Leung
parent
991d6bff38
commit
a0fe7ea2f0
@@ -60,6 +60,13 @@ class BlockStored(KVCacheEvent):
|
||||
medium: str | None
|
||||
lora_name: str | None
|
||||
|
||||
extra_keys: list[tuple[Any, ...] | None] | None = None
|
||||
"""Extra keys used in block hash computation, one entry per block in
|
||||
block_hashes. Each entry contains MM identifiers, LoRA name, cache_salt,
|
||||
prompt embedding hashes, etc. for that specific block. Exposed for external
|
||||
KV cache consumers to reconstruct block hashes.
|
||||
"""
|
||||
|
||||
def __hash__(self) -> int:
|
||||
return hash(
|
||||
(
|
||||
@@ -69,6 +76,7 @@ class BlockStored(KVCacheEvent):
|
||||
self.block_size,
|
||||
self.lora_id,
|
||||
self.medium,
|
||||
tuple(self.extra_keys) if self.extra_keys else None,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@@ -20,6 +20,7 @@ from vllm.v1.core.kv_cache_utils import (
|
||||
ExternalBlockHash,
|
||||
FreeKVCacheBlockQueue,
|
||||
KVCacheBlock,
|
||||
generate_block_hash_extra_keys,
|
||||
get_block_hash,
|
||||
make_block_hash_with_group_id,
|
||||
maybe_convert_block_hash,
|
||||
@@ -279,13 +280,31 @@ class BlockPool:
|
||||
block_hashes[num_cached_blocks - 1]
|
||||
)
|
||||
|
||||
# Calculate token range for the blocks being cached
|
||||
start_token_idx = num_cached_blocks * block_size
|
||||
end_token_idx = num_full_blocks * block_size
|
||||
|
||||
# Generate extra keys for each block individually.
|
||||
# Each block may have different extra_keys (e.g., different MM
|
||||
# features, or cache_salt only for the first block).
|
||||
# Skip null blocks to match the length of new_hashes.
|
||||
extra_keys_list: list[tuple[Any, ...] | None] = []
|
||||
curr_mm_idx = 0
|
||||
for i in range(num_cached_blocks, num_full_blocks):
|
||||
if blocks[i].is_null:
|
||||
continue
|
||||
block_start = i * block_size
|
||||
block_end = block_start + block_size
|
||||
extra_keys, curr_mm_idx = generate_block_hash_extra_keys(
|
||||
request, block_start, block_end, curr_mm_idx
|
||||
)
|
||||
extra_keys_list.append(extra_keys)
|
||||
|
||||
self.kv_event_queue.append(
|
||||
BlockStored(
|
||||
block_hashes=new_hashes,
|
||||
parent_block_hash=parent_block_hash,
|
||||
token_ids=request.all_token_ids[
|
||||
num_cached_blocks * block_size : num_full_blocks * block_size
|
||||
],
|
||||
token_ids=request.all_token_ids[start_token_idx:end_token_idx],
|
||||
block_size=block_size,
|
||||
lora_id=request.lora_request.adapter_id
|
||||
if request.lora_request
|
||||
@@ -294,6 +313,7 @@ class BlockPool:
|
||||
lora_name=request.lora_request.name
|
||||
if request.lora_request
|
||||
else None,
|
||||
extra_keys=extra_keys_list if extra_keys_list else None,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
"""KV-Cache Utilities."""
|
||||
|
||||
import copy
|
||||
import hashlib
|
||||
import os
|
||||
from collections import defaultdict
|
||||
from collections.abc import Callable, Iterable, Iterator, Sequence
|
||||
@@ -475,14 +476,19 @@ def _gen_prompt_embeds_extra_hash_keys(
|
||||
end_token_idx: The end token index of the block.
|
||||
|
||||
Returns:
|
||||
Return prompt embeddings data of the request if it has prompt embeds.
|
||||
Return empty list otherwise.
|
||||
Return a stable hash of the block prompt embeddings if prompt embeds
|
||||
are present. Return empty list otherwise.
|
||||
"""
|
||||
if request.prompt_embeds is None:
|
||||
return []
|
||||
block_prompt_embeds = request.prompt_embeds[start_token_idx:end_token_idx]
|
||||
embeds_bytes = tensor_data(block_prompt_embeds).tobytes()
|
||||
return [embeds_bytes]
|
||||
block_range = (start_token_idx, end_token_idx)
|
||||
embeds_hash = request._prompt_embeds_per_block_hashes.get(block_range)
|
||||
if embeds_hash is None:
|
||||
block_prompt_embeds = request.prompt_embeds[start_token_idx:end_token_idx]
|
||||
# Hash prompt embeds once per block and cache on request
|
||||
embeds_hash = hashlib.sha256(tensor_data(block_prompt_embeds)).digest()
|
||||
request._prompt_embeds_per_block_hashes[block_range] = embeds_hash
|
||||
return [embeds_hash]
|
||||
|
||||
|
||||
def generate_block_hash_extra_keys(
|
||||
@@ -490,7 +496,7 @@ def generate_block_hash_extra_keys(
|
||||
) -> tuple[tuple[Any, ...] | None, int]:
|
||||
"""Generate extra keys for the block hash. The extra keys can come from
|
||||
the multi-modal inputs, request specific metadata (e.g., LoRA names), and
|
||||
data from prompt embeddings.
|
||||
hashed data from prompt embeddings.
|
||||
|
||||
Args:
|
||||
request: The request object.
|
||||
|
||||
@@ -114,6 +114,9 @@ class Request:
|
||||
|
||||
self.prompt_token_ids = prompt_token_ids
|
||||
self.prompt_embeds = prompt_embeds
|
||||
# Cache per-block prompt-embed hashes to avoid rehashing the same
|
||||
# tensor slices when generating extra keys.
|
||||
self._prompt_embeds_per_block_hashes: dict[tuple[int, int], bytes] = {}
|
||||
self.num_prompt_tokens = length_from_prompt_token_ids_or_embeds(
|
||||
prompt_token_ids, prompt_embeds
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user