[Prefix Cache] Use LoRA name for consistent KV-cache block hashing (#27211)
Signed-off-by: Sage Ahrac <sagiahrak@gmail.com>
This commit is contained in:
@@ -373,7 +373,7 @@ def need_extra_keys(request: Request) -> bool:
|
||||
"""
|
||||
|
||||
# Multimodal requests need to include the MM hash.
|
||||
# LoRA requests need to include the LoRA ID.
|
||||
# LoRA requests need to include the LoRA name.
|
||||
# Request with provided cache salt need to include the salt.
|
||||
return (
|
||||
bool(request.mm_features)
|
||||
@@ -446,26 +446,26 @@ def _gen_mm_extra_hash_keys(
|
||||
return extra_keys, curr_mm_idx
|
||||
|
||||
|
||||
def _gen_lora_extra_hash_keys(request: Request) -> list[int]:
|
||||
def _gen_lora_extra_hash_keys(request: Request) -> list[str]:
|
||||
"""Generate extra keys related to LoRA for block hash computation.
|
||||
|
||||
Args:
|
||||
request: The request object.
|
||||
|
||||
Returns:
|
||||
Return LoRA id of the request if it is a LoRA request. Return empty
|
||||
Return LoRA name of the request if it is a LoRA request. Return empty
|
||||
list otherwise.
|
||||
"""
|
||||
if not request.lora_request:
|
||||
return []
|
||||
return [request.lora_request.lora_int_id]
|
||||
return [request.lora_request.lora_name]
|
||||
|
||||
|
||||
def generate_block_hash_extra_keys(
|
||||
request: Request, start_token_idx: int, end_token_idx: int, start_mm_idx: int
|
||||
) -> tuple[tuple[Any, ...] | None, int]:
|
||||
"""Generate extra keys for the block hash. The extra keys can come from
|
||||
the multi-modal inputs and request specific metadata (e.g., LoRA ID).
|
||||
the multi-modal inputs and request specific metadata (e.g., LoRA name).
|
||||
|
||||
Args:
|
||||
request: The request object.
|
||||
@@ -480,7 +480,7 @@ def generate_block_hash_extra_keys(
|
||||
mm_extra_keys, new_start_mm_idx = _gen_mm_extra_hash_keys(
|
||||
request, start_token_idx, end_token_idx, start_mm_idx
|
||||
)
|
||||
lora_extra_keys: list[int] = _gen_lora_extra_hash_keys(request)
|
||||
lora_extra_keys: list[str] = _gen_lora_extra_hash_keys(request)
|
||||
cache_salt_keys: list[str] = (
|
||||
[request.cache_salt] if (start_token_idx == 0 and request.cache_salt) else []
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user