[Bugfix] Disable fine-grained prefix-cache hits for incompatible hybrid KV layouts (#51843)
Signed-off-by: mgoin <mgoin64@gmail.com>
This commit is contained in:
@@ -24,6 +24,7 @@ from vllm.v1.kv_cache_interface import (
|
||||
KVCacheConfig,
|
||||
KVCacheGroupSpec,
|
||||
MambaSpec,
|
||||
SlidingWindowSpec,
|
||||
)
|
||||
|
||||
|
||||
@@ -1353,3 +1354,74 @@ def test_hybrid_partial_hit_with_eagle_stays_within_group_blocks():
|
||||
len(group) * block_size >= num_computed for group in computed_blocks.blocks
|
||||
)
|
||||
assert manager.allocate_slots(req1, 4, num_computed, computed_blocks) is not None
|
||||
|
||||
|
||||
def test_hybrid_sliding_window_group_disables_partial_hash_hits():
|
||||
hash_block_size = 2
|
||||
sliding_window_block_size = 2 * hash_block_size
|
||||
mamba_block_size = 2 * sliding_window_block_size
|
||||
kv_cache_config = KVCacheConfig(
|
||||
num_blocks=64,
|
||||
kv_cache_tensors=[],
|
||||
kv_cache_groups=[
|
||||
KVCacheGroupSpec(
|
||||
["full"],
|
||||
FullAttentionSpec(
|
||||
block_size=hash_block_size,
|
||||
num_kv_heads=1,
|
||||
head_size=1,
|
||||
dtype=torch.float32,
|
||||
),
|
||||
),
|
||||
KVCacheGroupSpec(
|
||||
["mamba"],
|
||||
MambaSpec(
|
||||
block_size=mamba_block_size,
|
||||
shapes=(1, 1),
|
||||
dtypes=(torch.float32,),
|
||||
mamba_cache_mode="align",
|
||||
),
|
||||
),
|
||||
KVCacheGroupSpec(
|
||||
["swa_draft"],
|
||||
SlidingWindowSpec(
|
||||
block_size=sliding_window_block_size,
|
||||
num_kv_heads=1,
|
||||
head_size=1,
|
||||
dtype=torch.float32,
|
||||
sliding_window=sliding_window_block_size,
|
||||
),
|
||||
is_eagle_group=True,
|
||||
),
|
||||
],
|
||||
)
|
||||
manager = make_kv_cache_manager(
|
||||
kv_cache_config=kv_cache_config,
|
||||
max_model_len=8192,
|
||||
enable_caching=True,
|
||||
hash_block_size=hash_block_size,
|
||||
use_eagle=True,
|
||||
)
|
||||
|
||||
tokens = list(range(3 * sliding_window_block_size))
|
||||
request = make_request("0", tokens, hash_block_size, sha256)
|
||||
computed_blocks, num_computed, _ = manager.get_computed_blocks(request)
|
||||
assert not manager.coordinator.enable_partial_hash_hits
|
||||
assert (
|
||||
manager.allocate_slots(request, mamba_block_size, num_computed, computed_blocks)
|
||||
is not None
|
||||
)
|
||||
request.num_computed_tokens = mamba_block_size
|
||||
manager.new_step_starts()
|
||||
assert manager.allocate_slots(request, len(tokens) - mamba_block_size) is not None
|
||||
request.num_computed_tokens = len(tokens)
|
||||
manager.free(request)
|
||||
manager.new_step_starts()
|
||||
|
||||
cached_request = make_request(
|
||||
"1", tokens + [len(tokens), len(tokens) + 1], hash_block_size, sha256
|
||||
)
|
||||
computed_blocks, num_computed, _ = manager.get_computed_blocks(cached_request)
|
||||
|
||||
assert num_computed == mamba_block_size
|
||||
assert len(computed_blocks.blocks[0]) * hash_block_size == num_computed
|
||||
|
||||
@@ -5,6 +5,7 @@ from collections.abc import Sequence
|
||||
from typing import NamedTuple
|
||||
|
||||
from vllm import envs
|
||||
from vllm.logger import init_logger
|
||||
from vllm.utils.math_utils import cdiv
|
||||
from vllm.v1.core.block_pool import BlockPool
|
||||
from vllm.v1.core.kv_cache_metrics import KVCacheMetricsCollector
|
||||
@@ -26,6 +27,8 @@ from vllm.v1.kv_cache_interface import (
|
||||
)
|
||||
from vllm.v1.request import Request
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def _validate_prefix_cache_retention_interval(
|
||||
retention_interval: int | None,
|
||||
@@ -62,6 +65,8 @@ class KVCacheCoordinator(ABC):
|
||||
Coordinate the KV cache of different KV cache groups.
|
||||
"""
|
||||
|
||||
enable_partial_hash_hits = False
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
kv_cache_config: KVCacheConfig,
|
||||
@@ -578,14 +583,29 @@ class HybridKVCacheCoordinator(KVCacheCoordinator):
|
||||
"full-attention and Mamba groups, got: "
|
||||
f"{type(g.kv_cache_spec).__name__}."
|
||||
)
|
||||
# Partial hash hits are limited to full-attention + mamba ("align")
|
||||
# without context parallelism.
|
||||
self.enable_partial_hash_hits = dcp_world_size == 1 and any(
|
||||
# Fine-grained hash hits require Mamba "align", no context
|
||||
# parallelism, and compatible cache managers in every group.
|
||||
has_partial_mamba_group = any(
|
||||
isinstance(g.kv_cache_spec, MambaSpec)
|
||||
and g.kv_cache_spec.mamba_cache_mode == "align"
|
||||
and g.kv_cache_spec.block_size > hash_block_size
|
||||
for g in kv_cache_config.kv_cache_groups
|
||||
)
|
||||
self.enable_partial_hash_hits = dcp_world_size == 1 and has_partial_mamba_group
|
||||
if self.enable_partial_hash_hits:
|
||||
unsupported_partial_hit_managers = {
|
||||
type(manager).__name__
|
||||
for manager in self.single_type_managers
|
||||
if not manager.supports_fine_grained_hash_lookup
|
||||
and manager.block_size != hash_block_size
|
||||
}
|
||||
if unsupported_partial_hit_managers:
|
||||
self.enable_partial_hash_hits = False
|
||||
logger.warning_once(
|
||||
"Disabling fine-grained prefix-cache hits because these KV "
|
||||
"cache managers require block-aligned lookups: %s.",
|
||||
", ".join(sorted(unsupported_partial_hit_managers)),
|
||||
)
|
||||
self.verify_and_split_kv_cache_groups()
|
||||
|
||||
@property
|
||||
|
||||
@@ -307,6 +307,7 @@ class Scheduler(SchedulerInterface):
|
||||
self.mamba_partial_cache_hit = (
|
||||
self.need_mamba_block_aligned_split
|
||||
and self.hash_block_size < self.block_size
|
||||
and self.kv_cache_manager.coordinator.enable_partial_hash_hits
|
||||
)
|
||||
|
||||
# Counts of non-empty steps scheduled / processed. update_from_output
|
||||
|
||||
Reference in New Issue
Block a user