[CORE] Support Prefix Caching with Prompt Embeds (#27219)
Signed-off-by: Andrew Sansom <andrew@protopia.ai>
This commit is contained in:
@@ -2,6 +2,7 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import importlib
|
||||
from collections.abc import Callable
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
@@ -32,6 +33,7 @@ from vllm.v1.core.kv_cache_utils import (
|
||||
init_none_hash,
|
||||
is_kv_cache_spec_uniform,
|
||||
make_block_hash_with_group_id,
|
||||
tensor_data,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
FullAttentionSpec,
|
||||
@@ -61,12 +63,13 @@ def _auto_init_hash_fn(request):
|
||||
|
||||
def make_request(
|
||||
request_id: str,
|
||||
prompt_token_ids: list[int],
|
||||
prompt_token_ids: list[int] | None,
|
||||
block_size: int = 3,
|
||||
hash_fn: Callable = hash,
|
||||
mm_positions: list[PlaceholderRange] | None = None,
|
||||
mm_hashes: list[str] | None = None,
|
||||
cache_salt: str | None = None,
|
||||
prompt_embeds: torch.Tensor | None = None,
|
||||
):
|
||||
mm_features = []
|
||||
if mm_positions is not None:
|
||||
@@ -90,6 +93,7 @@ def make_request(
|
||||
lora_request=None,
|
||||
cache_salt=cache_salt,
|
||||
block_hasher=get_request_block_hasher(block_size, hash_fn),
|
||||
prompt_embeds=prompt_embeds,
|
||||
)
|
||||
|
||||
|
||||
@@ -450,6 +454,52 @@ def test_generate_block_hash_extra_keys_cache_salt():
|
||||
assert next_mm_idx == 1
|
||||
|
||||
|
||||
def test_generate_block_hash_extra_keys_prompt_embeds():
|
||||
prompt_embeds = torch.randn(10, 3)
|
||||
request = make_request(
|
||||
request_id="0",
|
||||
prompt_token_ids=None,
|
||||
mm_positions=None,
|
||||
mm_hashes=None,
|
||||
prompt_embeds=prompt_embeds,
|
||||
)
|
||||
|
||||
# Test with prompt embeds for the first block
|
||||
extra_keys, _ = generate_block_hash_extra_keys(request, 0, 5, 0)
|
||||
expected_embeds = prompt_embeds[0:5]
|
||||
expected_bytes = kv_cache_utils.tensor_data(expected_embeds).tobytes()
|
||||
assert extra_keys == (expected_bytes,)
|
||||
|
||||
# Test with prompt embeds for the second block
|
||||
extra_keys, _ = generate_block_hash_extra_keys(request, 5, 10, 0)
|
||||
expected_embeds = prompt_embeds[5:10]
|
||||
expected_bytes = kv_cache_utils.tensor_data(expected_embeds).tobytes()
|
||||
assert extra_keys == (expected_bytes,)
|
||||
|
||||
|
||||
def test_generate_block_hash_extra_keys_different_prompt_embeds():
|
||||
prompt_embeds1 = torch.randn(10, 3)
|
||||
prompt_embeds2 = torch.randn(10, 3)
|
||||
request1 = make_request(
|
||||
request_id="0",
|
||||
prompt_token_ids=None,
|
||||
mm_positions=None,
|
||||
mm_hashes=None,
|
||||
prompt_embeds=prompt_embeds1,
|
||||
)
|
||||
request2 = make_request(
|
||||
request_id="1",
|
||||
prompt_token_ids=None,
|
||||
mm_positions=None,
|
||||
mm_hashes=None,
|
||||
prompt_embeds=prompt_embeds2,
|
||||
)
|
||||
|
||||
extra_keys1, _ = generate_block_hash_extra_keys(request1, 0, 5, 0)
|
||||
extra_keys2, _ = generate_block_hash_extra_keys(request2, 0, 5, 0)
|
||||
assert extra_keys1 != extra_keys2
|
||||
|
||||
|
||||
def test_generate_block_hash_extra_keys_lora():
|
||||
request = make_request(
|
||||
request_id="0",
|
||||
@@ -1556,3 +1606,88 @@ def test_merge_mla_spec():
|
||||
]
|
||||
with pytest.raises(AssertionError):
|
||||
kv_cache_specs[0].merge(kv_cache_specs)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("hash_fn", [sha256, sha256_cbor])
|
||||
def test_request_block_hasher_with_prompt_embeds(hash_fn: Callable[[Any], bytes]):
|
||||
block_size = 3
|
||||
num_tokens = 2 * block_size
|
||||
prompt_token_ids = [_ for _ in range(num_tokens)]
|
||||
hidden_size = 5
|
||||
prompt_embeds = torch.randn((num_tokens, hidden_size))
|
||||
|
||||
request = make_request(
|
||||
request_id="0",
|
||||
prompt_token_ids=prompt_token_ids,
|
||||
block_size=block_size,
|
||||
hash_fn=hash_fn,
|
||||
prompt_embeds=prompt_embeds,
|
||||
)
|
||||
|
||||
block_hashes = request.block_hashes
|
||||
assert len(block_hashes) == 2
|
||||
|
||||
block1_embeds_bytes = tensor_data(prompt_embeds[:block_size]).tobytes()
|
||||
expected_hash1 = hash_fn(
|
||||
(
|
||||
kv_cache_utils.NONE_HASH,
|
||||
tuple(prompt_token_ids[:block_size]),
|
||||
(block1_embeds_bytes,),
|
||||
)
|
||||
)
|
||||
assert block_hashes[0] == expected_hash1
|
||||
|
||||
block2_embeds_bytes = tensor_data(prompt_embeds[block_size:num_tokens]).tobytes()
|
||||
expected_hash2 = hash_fn(
|
||||
(
|
||||
block_hashes[0],
|
||||
tuple(prompt_token_ids[block_size:num_tokens]),
|
||||
(block2_embeds_bytes,),
|
||||
)
|
||||
)
|
||||
assert block_hashes[1] == expected_hash2
|
||||
|
||||
|
||||
@pytest.mark.parametrize("hash_fn", [sha256, sha256_cbor])
|
||||
def test_request_with_prompt_embeds_and_mm_inputs(hash_fn: Callable[[Any], bytes]):
|
||||
block_size = 3
|
||||
num_tokens = 2 * block_size
|
||||
prompt_token_ids = [_ for _ in range(num_tokens)]
|
||||
hidden_size = 5
|
||||
prompt_embeds = torch.randn((num_tokens, hidden_size))
|
||||
|
||||
request = make_request(
|
||||
request_id="0",
|
||||
prompt_token_ids=prompt_token_ids,
|
||||
block_size=block_size,
|
||||
hash_fn=hash_fn,
|
||||
mm_positions=[
|
||||
PlaceholderRange(offset=0, length=3),
|
||||
PlaceholderRange(offset=3, length=3),
|
||||
],
|
||||
mm_hashes=["hash1", "hash2"],
|
||||
prompt_embeds=prompt_embeds,
|
||||
)
|
||||
|
||||
block_hashes = request.block_hashes
|
||||
assert len(block_hashes) == 2
|
||||
|
||||
block1_embeds_bytes = tensor_data(prompt_embeds[:block_size]).tobytes()
|
||||
expected_hash1 = hash_fn(
|
||||
(
|
||||
kv_cache_utils.NONE_HASH,
|
||||
tuple(prompt_token_ids[:block_size]),
|
||||
("hash1", block1_embeds_bytes),
|
||||
)
|
||||
)
|
||||
assert block_hashes[0] == expected_hash1
|
||||
|
||||
block2_embeds_bytes = tensor_data(prompt_embeds[block_size:num_tokens]).tobytes()
|
||||
expected_hash2 = hash_fn(
|
||||
(
|
||||
block_hashes[0],
|
||||
tuple(prompt_token_ids[block_size:num_tokens]),
|
||||
("hash2", block2_embeds_bytes),
|
||||
)
|
||||
)
|
||||
assert block_hashes[1] == expected_hash2
|
||||
|
||||
Reference in New Issue
Block a user