[Prefix Cache] Use LoRA name for consistent KV-cache block hashing (#27211)

Signed-off-by: Sage Ahrac <sagiahrak@gmail.com>
This commit is contained in:
Sage
2025-10-22 18:13:03 +00:00
committed by GitHub
parent 1cb8c6c5fe
commit 1651003c35
2 changed files with 25 additions and 6 deletions
+6 -6
View File
@@ -373,7 +373,7 @@ def need_extra_keys(request: Request) -> bool:
"""
# Multimodal requests need to include the MM hash.
# LoRA requests need to include the LoRA ID.
# LoRA requests need to include the LoRA name.
# Request with provided cache salt need to include the salt.
return (
bool(request.mm_features)
@@ -446,26 +446,26 @@ def _gen_mm_extra_hash_keys(
return extra_keys, curr_mm_idx
def _gen_lora_extra_hash_keys(request: Request) -> list[int]:
def _gen_lora_extra_hash_keys(request: Request) -> list[str]:
"""Generate extra keys related to LoRA for block hash computation.
Args:
request: The request object.
Returns:
Return LoRA id of the request if it is a LoRA request. Return empty
Return LoRA name of the request if it is a LoRA request. Return empty
list otherwise.
"""
if not request.lora_request:
return []
return [request.lora_request.lora_int_id]
return [request.lora_request.lora_name]
def generate_block_hash_extra_keys(
request: Request, start_token_idx: int, end_token_idx: int, start_mm_idx: int
) -> tuple[tuple[Any, ...] | None, int]:
"""Generate extra keys for the block hash. The extra keys can come from
the multi-modal inputs and request specific metadata (e.g., LoRA ID).
the multi-modal inputs and request specific metadata (e.g., LoRA name).
Args:
request: The request object.
@@ -480,7 +480,7 @@ def generate_block_hash_extra_keys(
mm_extra_keys, new_start_mm_idx = _gen_mm_extra_hash_keys(
request, start_token_idx, end_token_idx, start_mm_idx
)
lora_extra_keys: list[int] = _gen_lora_extra_hash_keys(request)
lora_extra_keys: list[str] = _gen_lora_extra_hash_keys(request)
cache_salt_keys: list[str] = (
[request.cache_salt] if (start_token_idx == 0 and request.cache_salt) else []
)