feat(kv-events): emit KV cache metadata (#40984)
Signed-off-by: PeaBrane <yanrpei@gmail.com>
This commit is contained in:
@@ -9,7 +9,10 @@ from vllm.distributed.kv_events import BlockRemoved, BlockStored
|
||||
_FAKE_HASH: bytes = b"\xab" * 32
|
||||
|
||||
|
||||
def _make_block_stored(group_idx: int | None = None) -> BlockStored:
|
||||
def _make_block_stored(
|
||||
group_idx: int | None = None,
|
||||
kv_cache_spec_sliding_window: int | None = None,
|
||||
) -> BlockStored:
|
||||
return BlockStored(
|
||||
block_hashes=[_FAKE_HASH],
|
||||
parent_block_hash=None,
|
||||
@@ -19,10 +22,13 @@ def _make_block_stored(group_idx: int | None = None) -> BlockStored:
|
||||
medium="GPU",
|
||||
lora_name=None,
|
||||
group_idx=group_idx,
|
||||
kv_cache_spec_sliding_window=kv_cache_spec_sliding_window,
|
||||
)
|
||||
|
||||
|
||||
def _make_block_removed(group_idx: int | None = None) -> BlockRemoved:
|
||||
def _make_block_removed(
|
||||
group_idx: int | None = None,
|
||||
) -> BlockRemoved:
|
||||
return BlockRemoved(
|
||||
block_hashes=[_FAKE_HASH],
|
||||
medium="GPU",
|
||||
@@ -72,3 +78,9 @@ def test_block_removed_hash_same_for_equal_group_idx():
|
||||
event_a = _make_block_removed(group_idx=1)
|
||||
event_b = _make_block_removed(group_idx=1)
|
||||
assert hash(event_a) == hash(event_b)
|
||||
|
||||
|
||||
def test_block_stored_hash_differs_by_sliding_window():
|
||||
event_a = _make_block_stored(group_idx=1, kv_cache_spec_sliding_window=128)
|
||||
event_b = _make_block_stored(group_idx=1, kv_cache_spec_sliding_window=256)
|
||||
assert hash(event_a) != hash(event_b)
|
||||
|
||||
@@ -43,11 +43,16 @@ from vllm.v1.kv_cache_interface import (
|
||||
KVCacheConfig,
|
||||
KVCacheGroupSpec,
|
||||
KVCacheSpec,
|
||||
KVCacheSpecKind,
|
||||
KVCacheTensor,
|
||||
MambaSpec,
|
||||
MLAAttentionSpec,
|
||||
SinkFullAttentionSpec,
|
||||
SlidingWindowMLASpec,
|
||||
SlidingWindowSpec,
|
||||
UniformTypeKVCacheSpecs,
|
||||
get_kv_cache_spec_kind,
|
||||
get_kv_cache_spec_sliding_window,
|
||||
)
|
||||
from vllm.v1.metrics.stats import CachingMetrics, PrefixCacheStats
|
||||
from vllm.v1.request import Request
|
||||
@@ -1865,6 +1870,149 @@ def new_mla_spec(cache_dtype_str=None):
|
||||
)
|
||||
|
||||
|
||||
def test_get_kv_cache_spec_kind_prefers_specific_attention_subclasses():
|
||||
assert get_kv_cache_spec_kind(new_mla_spec()) == KVCacheSpecKind.MLA_ATTENTION
|
||||
|
||||
sliding_window_mla_spec = SlidingWindowMLASpec(
|
||||
block_size=16,
|
||||
num_kv_heads=1,
|
||||
head_size=576,
|
||||
dtype=torch.float32,
|
||||
sliding_window=128,
|
||||
)
|
||||
assert (
|
||||
get_kv_cache_spec_kind(sliding_window_mla_spec)
|
||||
== KVCacheSpecKind.SLIDING_WINDOW_MLA
|
||||
)
|
||||
|
||||
sink_full_attention_spec = SinkFullAttentionSpec(
|
||||
block_size=16,
|
||||
num_kv_heads=1,
|
||||
head_size=64,
|
||||
dtype=torch.float32,
|
||||
sink_len=4,
|
||||
)
|
||||
assert (
|
||||
get_kv_cache_spec_kind(sink_full_attention_spec)
|
||||
== KVCacheSpecKind.SINK_FULL_ATTENTION
|
||||
)
|
||||
|
||||
|
||||
def test_get_kv_cache_spec_kind_unwraps_uniform_type_specs():
|
||||
uniform_mla_spec = UniformTypeKVCacheSpecs(
|
||||
block_size=16,
|
||||
kv_cache_specs={
|
||||
"layer_1": new_mla_spec(),
|
||||
"layer_2": new_mla_spec(cache_dtype_str="fp8"),
|
||||
},
|
||||
)
|
||||
assert get_kv_cache_spec_kind(uniform_mla_spec) == KVCacheSpecKind.MLA_ATTENTION
|
||||
|
||||
uniform_swa_mla_spec = UniformTypeKVCacheSpecs(
|
||||
block_size=16,
|
||||
kv_cache_specs={
|
||||
"layer_1": SlidingWindowMLASpec(
|
||||
block_size=16,
|
||||
num_kv_heads=1,
|
||||
head_size=576,
|
||||
dtype=torch.float32,
|
||||
sliding_window=128,
|
||||
),
|
||||
"layer_2": SlidingWindowMLASpec(
|
||||
block_size=16,
|
||||
num_kv_heads=1,
|
||||
head_size=1024,
|
||||
dtype=torch.float32,
|
||||
sliding_window=128,
|
||||
),
|
||||
},
|
||||
)
|
||||
assert (
|
||||
get_kv_cache_spec_kind(uniform_swa_mla_spec)
|
||||
== KVCacheSpecKind.SLIDING_WINDOW_MLA
|
||||
)
|
||||
|
||||
|
||||
def test_get_kv_cache_spec_kind_unknown_for_mixed_uniform_type_specs():
|
||||
uniform_mixed_spec = UniformTypeKVCacheSpecs(
|
||||
block_size=16,
|
||||
kv_cache_specs={
|
||||
"layer_1": new_mla_spec(),
|
||||
"layer_2": SlidingWindowMLASpec(
|
||||
block_size=16,
|
||||
num_kv_heads=1,
|
||||
head_size=576,
|
||||
dtype=torch.float32,
|
||||
sliding_window=128,
|
||||
),
|
||||
},
|
||||
)
|
||||
assert get_kv_cache_spec_kind(uniform_mixed_spec) == KVCacheSpecKind.UNKNOWN
|
||||
|
||||
|
||||
def test_get_kv_cache_spec_sliding_window_reads_windowed_specs():
|
||||
full_attention_spec = FullAttentionSpec(
|
||||
block_size=16,
|
||||
num_kv_heads=1,
|
||||
head_size=64,
|
||||
dtype=torch.float32,
|
||||
)
|
||||
sliding_window_spec = SlidingWindowSpec(
|
||||
block_size=16,
|
||||
num_kv_heads=1,
|
||||
head_size=64,
|
||||
dtype=torch.float32,
|
||||
sliding_window=128,
|
||||
)
|
||||
|
||||
assert get_kv_cache_spec_sliding_window(full_attention_spec) is None
|
||||
assert get_kv_cache_spec_sliding_window(sliding_window_spec) == 128
|
||||
|
||||
|
||||
def test_get_kv_cache_spec_sliding_window_unwraps_uniform_type_specs():
|
||||
uniform_window_spec = UniformTypeKVCacheSpecs(
|
||||
block_size=16,
|
||||
kv_cache_specs={
|
||||
"layer_1": SlidingWindowSpec(
|
||||
block_size=16,
|
||||
num_kv_heads=1,
|
||||
head_size=64,
|
||||
dtype=torch.float32,
|
||||
sliding_window=128,
|
||||
),
|
||||
"layer_2": SlidingWindowSpec(
|
||||
block_size=16,
|
||||
num_kv_heads=2,
|
||||
head_size=64,
|
||||
dtype=torch.float32,
|
||||
sliding_window=128,
|
||||
),
|
||||
},
|
||||
)
|
||||
mixed_window_spec = UniformTypeKVCacheSpecs(
|
||||
block_size=16,
|
||||
kv_cache_specs={
|
||||
"layer_1": SlidingWindowSpec(
|
||||
block_size=16,
|
||||
num_kv_heads=1,
|
||||
head_size=64,
|
||||
dtype=torch.float32,
|
||||
sliding_window=128,
|
||||
),
|
||||
"layer_2": SlidingWindowSpec(
|
||||
block_size=16,
|
||||
num_kv_heads=1,
|
||||
head_size=64,
|
||||
dtype=torch.float32,
|
||||
sliding_window=256,
|
||||
),
|
||||
},
|
||||
)
|
||||
|
||||
assert get_kv_cache_spec_sliding_window(uniform_window_spec) == 128
|
||||
assert get_kv_cache_spec_sliding_window(mixed_window_spec) is None
|
||||
|
||||
|
||||
def test_merge_mla_spec():
|
||||
kv_cache_specs = [
|
||||
new_mla_spec(),
|
||||
|
||||
@@ -8,6 +8,7 @@ from collections.abc import Callable
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
import vllm.v1.core.kv_cache_manager as kv_cache_manager
|
||||
import vllm.v1.core.kv_cache_utils as kv_cache_utils
|
||||
from vllm.distributed.kv_events import AllBlocksCleared, BlockRemoved, BlockStored
|
||||
from vllm.lora.request import LoRARequest
|
||||
@@ -35,6 +36,7 @@ from vllm.v1.kv_cache_interface import (
|
||||
FullAttentionSpec,
|
||||
KVCacheConfig,
|
||||
KVCacheGroupSpec,
|
||||
KVCacheSpecKind,
|
||||
MambaSpec,
|
||||
SlidingWindowSpec,
|
||||
)
|
||||
@@ -1933,6 +1935,7 @@ def test_kv_cache_events(blocks_to_cache: int):
|
||||
== len(manager.block_pool.cached_block_hash_to_block)
|
||||
)
|
||||
assert len(block.token_ids) == block.block_size * len(block.block_hashes)
|
||||
assert block.kv_cache_spec_kind == KVCacheSpecKind.FULL_ATTENTION.value
|
||||
assert len(manager.block_pool.kv_event_queue) == 0
|
||||
|
||||
stored_block_hash = block.block_hashes
|
||||
@@ -1946,6 +1949,7 @@ def test_kv_cache_events(blocks_to_cache: int):
|
||||
events = manager.take_events()
|
||||
|
||||
for blocks in events[:-1]:
|
||||
assert isinstance(blocks, BlockRemoved)
|
||||
assert blocks.block_hashes[0] in stored_block_hash
|
||||
assert len(events) == blocks_to_cache + 1
|
||||
assert isinstance(events[-2], BlockRemoved)
|
||||
@@ -2022,6 +2026,8 @@ def test_null_parent_block_hash():
|
||||
]
|
||||
assert event.block_hashes == expected_new_hashes
|
||||
assert event.group_idx == kv_cache_group_id
|
||||
assert event.kv_cache_spec_kind is None
|
||||
assert event.kv_cache_spec_sliding_window is None
|
||||
|
||||
# Ensure we didn't accidentally assign a hash to the null block.
|
||||
assert pool.null_block.block_hash is None
|
||||
@@ -2095,12 +2101,14 @@ def test_block_stored_event_group_idx(group_id: int):
|
||||
block_size = 4
|
||||
num_tokens = block_size * 2
|
||||
|
||||
pool = BlockPool(
|
||||
num_gpu_blocks=5,
|
||||
manager = KVCacheManager(
|
||||
make_kv_cache_config_three_types(block_size, num_blocks=5),
|
||||
max_model_len=8192,
|
||||
enable_caching=True,
|
||||
hash_block_size=block_size,
|
||||
enable_kv_cache_events=True,
|
||||
hash_block_size=block_size,
|
||||
)
|
||||
pool = manager.block_pool
|
||||
|
||||
req = make_request(
|
||||
"req_grp_idx",
|
||||
@@ -2119,10 +2127,26 @@ def test_block_stored_event_group_idx(group_id: int):
|
||||
kv_cache_group_id=group_id,
|
||||
)
|
||||
|
||||
events = pool.take_events()
|
||||
events = manager.take_events()
|
||||
assert len(events) == 1
|
||||
assert isinstance(events[0], BlockStored)
|
||||
assert events[0].group_idx == group_id
|
||||
assert (
|
||||
events[0].kv_cache_spec_kind
|
||||
== [
|
||||
KVCacheSpecKind.FULL_ATTENTION.value,
|
||||
KVCacheSpecKind.SLIDING_WINDOW.value,
|
||||
KVCacheSpecKind.MAMBA.value,
|
||||
][group_id]
|
||||
)
|
||||
assert (
|
||||
events[0].kv_cache_spec_sliding_window
|
||||
== [
|
||||
None,
|
||||
2 * block_size,
|
||||
None,
|
||||
][group_id]
|
||||
)
|
||||
|
||||
|
||||
def test_block_stored_event_group_idx_multiple_groups():
|
||||
@@ -2137,13 +2161,38 @@ def test_block_stored_event_group_idx_multiple_groups():
|
||||
block_size = 4
|
||||
num_tokens = block_size * 2
|
||||
|
||||
# null block + 4 usable (2 per group)
|
||||
pool = BlockPool(
|
||||
num_gpu_blocks=5,
|
||||
manager = KVCacheManager(
|
||||
KVCacheConfig(
|
||||
num_blocks=5,
|
||||
kv_cache_tensors=[],
|
||||
kv_cache_groups=[
|
||||
KVCacheGroupSpec(
|
||||
["layer1"],
|
||||
FullAttentionSpec(
|
||||
block_size=block_size,
|
||||
num_kv_heads=1,
|
||||
head_size=1,
|
||||
dtype=torch.float32,
|
||||
),
|
||||
),
|
||||
KVCacheGroupSpec(
|
||||
["layer2"],
|
||||
SlidingWindowSpec(
|
||||
block_size=block_size,
|
||||
num_kv_heads=1,
|
||||
head_size=1,
|
||||
dtype=torch.float32,
|
||||
sliding_window=128,
|
||||
),
|
||||
),
|
||||
],
|
||||
),
|
||||
max_model_len=8192,
|
||||
enable_caching=True,
|
||||
hash_block_size=block_size,
|
||||
enable_kv_cache_events=True,
|
||||
hash_block_size=block_size,
|
||||
)
|
||||
pool = manager.block_pool
|
||||
|
||||
req = make_request(
|
||||
"req_multi_grp",
|
||||
@@ -2174,12 +2223,52 @@ def test_block_stored_event_group_idx_multiple_groups():
|
||||
kv_cache_group_id=1,
|
||||
)
|
||||
|
||||
events = pool.take_events()
|
||||
events = manager.take_events()
|
||||
assert len(events) == 2
|
||||
assert isinstance(events[0], BlockStored)
|
||||
assert events[0].group_idx == 0
|
||||
assert events[0].kv_cache_spec_kind == KVCacheSpecKind.FULL_ATTENTION.value
|
||||
assert events[0].kv_cache_spec_sliding_window is None
|
||||
assert isinstance(events[1], BlockStored)
|
||||
assert events[1].group_idx == 1
|
||||
assert events[1].kv_cache_spec_kind == KVCacheSpecKind.SLIDING_WINDOW.value
|
||||
assert events[1].kv_cache_spec_sliding_window == 128
|
||||
|
||||
|
||||
def test_block_stored_event_group_idx_out_of_bounds(monkeypatch):
|
||||
"""Out-of-range group_idx events are returned without metadata annotation."""
|
||||
block_size = 4
|
||||
manager = KVCacheManager(
|
||||
make_kv_cache_config(block_size, num_blocks=5),
|
||||
max_model_len=8192,
|
||||
enable_caching=True,
|
||||
enable_kv_cache_events=True,
|
||||
hash_block_size=block_size,
|
||||
)
|
||||
event = BlockStored(
|
||||
block_hashes=[1],
|
||||
parent_block_hash=None,
|
||||
token_ids=list(range(block_size)),
|
||||
block_size=block_size,
|
||||
lora_id=None,
|
||||
medium=None,
|
||||
lora_name=None,
|
||||
group_idx=1,
|
||||
)
|
||||
manager.block_pool.kv_event_queue.append(event)
|
||||
warnings = []
|
||||
|
||||
def collect_warning(message, *args, **kwargs):
|
||||
del kwargs
|
||||
warnings.append(message % args if args else message)
|
||||
|
||||
monkeypatch.setattr(kv_cache_manager.logger, "warning", collect_warning)
|
||||
events = manager.take_events()
|
||||
|
||||
assert events == [event]
|
||||
assert event.kv_cache_spec_kind is None
|
||||
assert event.kv_cache_spec_sliding_window is None
|
||||
assert warnings == ["Group index `1` not in KV cache metadata"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("group_id", [0, 1, 2])
|
||||
|
||||
Reference in New Issue
Block a user