[Core] Include the LoRA path in prefix-cache block hashes (#59335)
Signed-off-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
d7c55116ac
commit
c055c1e075
@@ -1022,7 +1022,7 @@ def test_generate_block_hash_extra_keys_lora():
|
||||
)
|
||||
|
||||
extra_keys, _ = generate_block_hash_extra_keys(request, 0, 3, 0)
|
||||
assert extra_keys == (("lora", "test_lora_adapter"),)
|
||||
assert extra_keys == (("lora", "test_lora_adapter", "/path/to/lora"),)
|
||||
|
||||
request.lora_request = None
|
||||
extra_keys, _ = generate_block_hash_extra_keys(request, 0, 3, 0)
|
||||
@@ -1048,6 +1048,28 @@ def test_lora_name_and_cache_salt_block_hashes_do_not_collide(hash_fn):
|
||||
assert lora_req.block_hashes[0] != salted_req.block_hashes[0]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("hash_fn", [sha256, sha256_cbor])
|
||||
def test_lora_path_change_changes_block_hashes(hash_fn):
|
||||
"""Re-pointing a LoRA name at another adapter must not reuse its blocks."""
|
||||
|
||||
def make_lora_request(lora_path: str) -> Request:
|
||||
return Request(
|
||||
request_id=lora_path,
|
||||
prompt_token_ids=[0, 1, 2],
|
||||
sampling_params=SamplingParams(max_tokens=1),
|
||||
pooling_params=None,
|
||||
lora_request=LoRARequest(
|
||||
lora_name="foo", lora_int_id=1, lora_path=lora_path
|
||||
),
|
||||
block_hasher=get_request_block_hasher(3, hash_fn),
|
||||
)
|
||||
|
||||
first = make_lora_request("/path/to/lora_a")
|
||||
second = make_lora_request("/path/to/lora_b")
|
||||
|
||||
assert first.block_hashes[0] != second.block_hashes[0]
|
||||
|
||||
|
||||
def test_to_event_extra_keys_keeps_untagged_event_format():
|
||||
"""KV events keep publishing the extra-key shapes consumers already parse."""
|
||||
request = make_request(
|
||||
@@ -1064,7 +1086,7 @@ def test_to_event_extra_keys_keeps_untagged_event_format():
|
||||
extra_keys, _ = generate_block_hash_extra_keys(request, 0, 5, 0)
|
||||
|
||||
assert extra_keys == (
|
||||
("lora", "adapter"),
|
||||
("lora", "adapter", "/path/to/lora"),
|
||||
("mm", "hash1", 2),
|
||||
("cache_salt", "salt"),
|
||||
)
|
||||
|
||||
@@ -212,6 +212,16 @@ class OpenAIServingModels:
|
||||
status_code=HTTPStatus.INTERNAL_SERVER_ERROR,
|
||||
)
|
||||
|
||||
previous = self.lora_requests.get(lora_name)
|
||||
if previous is not None and previous.lora_path == lora_path:
|
||||
logger.warning(
|
||||
"Reloaded LoRA adapter '%s' in place from the same path '%s'. "
|
||||
"Prefix-cache blocks computed with its previous weights are "
|
||||
"not invalidated; load changed weights from a new path to "
|
||||
"avoid reusing them.",
|
||||
lora_name,
|
||||
lora_path,
|
||||
)
|
||||
self.lora_requests[lora_name] = lora_request
|
||||
logger.info(
|
||||
"Loaded new LoRA adapter: name '%s', path '%s'", lora_name, lora_path
|
||||
|
||||
@@ -565,20 +565,24 @@ def _gen_mm_extra_hash_keys(
|
||||
return extra_keys, curr_mm_idx
|
||||
|
||||
|
||||
def _gen_lora_extra_hash_keys(request: Request) -> list[tuple[str, str]]:
|
||||
def _gen_lora_extra_hash_keys(request: Request) -> list[tuple[str, str, str]]:
|
||||
"""Generate extra keys related to LoRA for block hash computation.
|
||||
|
||||
The adapter path is included so that re-pointing a LoRA name at a different
|
||||
adapter does not reuse KV computed with the previous one.
|
||||
|
||||
Args:
|
||||
request: The request object.
|
||||
|
||||
Returns:
|
||||
Return LoRA name of the request if it is a LoRA request. Return empty
|
||||
list otherwise.
|
||||
Return the LoRA name and path of the request if it is a LoRA request.
|
||||
Return empty list otherwise.
|
||||
|
||||
"""
|
||||
if not request.lora_request:
|
||||
lora_request = request.lora_request
|
||||
if not lora_request:
|
||||
return []
|
||||
return [("lora", request.lora_request.lora_name)]
|
||||
return [("lora", lora_request.lora_name, lora_request.lora_path)]
|
||||
|
||||
|
||||
def _gen_prompt_embeds_extra_hash_keys(
|
||||
|
||||
Reference in New Issue
Block a user