[V1][Spec Decode] KV cache slots for eagle heads (#16370)
Signed-off-by: LiuXiaoxuanPKU <lilyliupku@gmail.com>
This commit is contained in:
@@ -7,6 +7,7 @@ from vllm.config import ModelConfig, SchedulerConfig, VllmConfig
|
||||
from vllm.multimodal.inputs import MultiModalKwargs, PlaceholderRange
|
||||
from vllm.sampling_params import SamplingParams
|
||||
from vllm.utils import GiB_bytes, sha256
|
||||
from vllm.v1.core.kv_cache_manager import KVCacheManager
|
||||
# disable yapf here as it formats differently than isort such that both fail
|
||||
# yapf: disable
|
||||
from vllm.v1.core.kv_cache_utils import (NONE_HASH, BlockHashType,
|
||||
@@ -48,6 +49,18 @@ def make_request(request_id,
|
||||
)
|
||||
|
||||
|
||||
def new_kv_cache_spec(block_size=16,
|
||||
num_kv_heads=2,
|
||||
head_size=64,
|
||||
dtype=torch.float32,
|
||||
use_mla=False):
|
||||
return FullAttentionSpec(block_size=block_size,
|
||||
num_kv_heads=num_kv_heads,
|
||||
head_size=head_size,
|
||||
dtype=dtype,
|
||||
use_mla=use_mla)
|
||||
|
||||
|
||||
def test_none_hash():
|
||||
assert NONE_HASH is not None
|
||||
assert isinstance(NONE_HASH, int)
|
||||
@@ -327,18 +340,6 @@ def test_metrics():
|
||||
|
||||
|
||||
def test_unify_kv_cache_configs():
|
||||
|
||||
def new_kv_cache_spec(block_size=16,
|
||||
num_kv_heads=2,
|
||||
head_size=64,
|
||||
dtype=torch.float32,
|
||||
use_mla=False):
|
||||
return FullAttentionSpec(block_size=block_size,
|
||||
num_kv_heads=num_kv_heads,
|
||||
head_size=head_size,
|
||||
dtype=dtype,
|
||||
use_mla=use_mla)
|
||||
|
||||
same_kv_cache_config = [
|
||||
KVCacheConfig(
|
||||
num_blocks=10,
|
||||
@@ -470,3 +471,64 @@ def test_estimate_max_model_len(model_id, max_model_len,
|
||||
estimated_max_len = estimate_max_model_len(vllm_config, kv_cache_spec,
|
||||
8 * GiB_bytes)
|
||||
assert estimated_max_len == want_estimated_max_len
|
||||
|
||||
|
||||
def test_allocate_with_lookahead():
|
||||
"""Verify that lookahead tokens correctly affect block allocation"""
|
||||
block_size = 4
|
||||
config = KVCacheConfig(
|
||||
num_blocks=10,
|
||||
tensors={
|
||||
"layer1": KVCacheTensor(100),
|
||||
},
|
||||
kv_cache_groups=[
|
||||
KVCacheGroupSpec(["layer1"],
|
||||
new_kv_cache_spec(block_size=block_size)),
|
||||
],
|
||||
)
|
||||
|
||||
request = make_request(
|
||||
request_id=0,
|
||||
prompt_token_ids=[],
|
||||
mm_positions=None,
|
||||
mm_hashes=None,
|
||||
)
|
||||
|
||||
# Test case 1: Requires additional lookahead tokens
|
||||
kv_cache_manager = KVCacheManager(kv_cache_config=config,
|
||||
max_model_len=100,
|
||||
num_preallocate_tokens=0)
|
||||
blocks = kv_cache_manager.allocate_slots(
|
||||
request,
|
||||
num_tokens=3,
|
||||
num_lookahead_tokens=2, # Total required: 3+2=5 tokens
|
||||
)
|
||||
assert len(blocks) == 2 # ceil(5/4)=2 blocks
|
||||
|
||||
# Test case 2: With precomputed blocks
|
||||
kv_cache_manager = KVCacheManager(kv_cache_config=config,
|
||||
max_model_len=100,
|
||||
num_preallocate_tokens=4)
|
||||
# num_preallocate_blocks = 4 // 4 - 2 // 4 = 1
|
||||
# required_blocks = ceil((3 + 2) /4) = 2
|
||||
# total_blocks = 1 + 2 = 3
|
||||
blocks = kv_cache_manager.allocate_slots(
|
||||
request,
|
||||
num_tokens=3,
|
||||
num_lookahead_tokens=2,
|
||||
)
|
||||
assert len(blocks) == 3
|
||||
|
||||
# Test case 3: With precomputed blocks
|
||||
# num_preallocate_blocks = 4 // 4 - 4 // 4 = 0
|
||||
# required_blocks = ceil((3 + 4) / 4) = 2
|
||||
# total_blocks = 0 + 2 = 2
|
||||
kv_cache_manager = KVCacheManager(kv_cache_config=config,
|
||||
max_model_len=100,
|
||||
num_preallocate_tokens=4)
|
||||
blocks = kv_cache_manager.allocate_slots(
|
||||
request,
|
||||
num_tokens=3,
|
||||
num_lookahead_tokens=4,
|
||||
)
|
||||
assert len(blocks) == 2
|
||||
|
||||
Reference in New Issue
Block a user