feat(kv-events): emit KV cache metadata (#40984)

Signed-off-by: PeaBrane <yanrpei@gmail.com>
This commit is contained in:
Yan Ru Pei
2026-05-12 15:58:48 +00:00
committed by GitHub
parent c8a6e272e0
commit bcb9c133ba
8 changed files with 381 additions and 21 deletions
+14 -2
View File
@@ -9,7 +9,10 @@ from vllm.distributed.kv_events import BlockRemoved, BlockStored
_FAKE_HASH: bytes = b"\xab" * 32
def _make_block_stored(group_idx: int | None = None) -> BlockStored:
def _make_block_stored(
group_idx: int | None = None,
kv_cache_spec_sliding_window: int | None = None,
) -> BlockStored:
return BlockStored(
block_hashes=[_FAKE_HASH],
parent_block_hash=None,
@@ -19,10 +22,13 @@ def _make_block_stored(group_idx: int | None = None) -> BlockStored:
medium="GPU",
lora_name=None,
group_idx=group_idx,
kv_cache_spec_sliding_window=kv_cache_spec_sliding_window,
)
def _make_block_removed(group_idx: int | None = None) -> BlockRemoved:
def _make_block_removed(
group_idx: int | None = None,
) -> BlockRemoved:
return BlockRemoved(
block_hashes=[_FAKE_HASH],
medium="GPU",
@@ -72,3 +78,9 @@ def test_block_removed_hash_same_for_equal_group_idx():
event_a = _make_block_removed(group_idx=1)
event_b = _make_block_removed(group_idx=1)
assert hash(event_a) == hash(event_b)
def test_block_stored_hash_differs_by_sliding_window():
event_a = _make_block_stored(group_idx=1, kv_cache_spec_sliding_window=128)
event_b = _make_block_stored(group_idx=1, kv_cache_spec_sliding_window=256)
assert hash(event_a) != hash(event_b)
+148
View File
@@ -43,11 +43,16 @@ from vllm.v1.kv_cache_interface import (
KVCacheConfig,
KVCacheGroupSpec,
KVCacheSpec,
KVCacheSpecKind,
KVCacheTensor,
MambaSpec,
MLAAttentionSpec,
SinkFullAttentionSpec,
SlidingWindowMLASpec,
SlidingWindowSpec,
UniformTypeKVCacheSpecs,
get_kv_cache_spec_kind,
get_kv_cache_spec_sliding_window,
)
from vllm.v1.metrics.stats import CachingMetrics, PrefixCacheStats
from vllm.v1.request import Request
@@ -1865,6 +1870,149 @@ def new_mla_spec(cache_dtype_str=None):
)
def test_get_kv_cache_spec_kind_prefers_specific_attention_subclasses():
assert get_kv_cache_spec_kind(new_mla_spec()) == KVCacheSpecKind.MLA_ATTENTION
sliding_window_mla_spec = SlidingWindowMLASpec(
block_size=16,
num_kv_heads=1,
head_size=576,
dtype=torch.float32,
sliding_window=128,
)
assert (
get_kv_cache_spec_kind(sliding_window_mla_spec)
== KVCacheSpecKind.SLIDING_WINDOW_MLA
)
sink_full_attention_spec = SinkFullAttentionSpec(
block_size=16,
num_kv_heads=1,
head_size=64,
dtype=torch.float32,
sink_len=4,
)
assert (
get_kv_cache_spec_kind(sink_full_attention_spec)
== KVCacheSpecKind.SINK_FULL_ATTENTION
)
def test_get_kv_cache_spec_kind_unwraps_uniform_type_specs():
uniform_mla_spec = UniformTypeKVCacheSpecs(
block_size=16,
kv_cache_specs={
"layer_1": new_mla_spec(),
"layer_2": new_mla_spec(cache_dtype_str="fp8"),
},
)
assert get_kv_cache_spec_kind(uniform_mla_spec) == KVCacheSpecKind.MLA_ATTENTION
uniform_swa_mla_spec = UniformTypeKVCacheSpecs(
block_size=16,
kv_cache_specs={
"layer_1": SlidingWindowMLASpec(
block_size=16,
num_kv_heads=1,
head_size=576,
dtype=torch.float32,
sliding_window=128,
),
"layer_2": SlidingWindowMLASpec(
block_size=16,
num_kv_heads=1,
head_size=1024,
dtype=torch.float32,
sliding_window=128,
),
},
)
assert (
get_kv_cache_spec_kind(uniform_swa_mla_spec)
== KVCacheSpecKind.SLIDING_WINDOW_MLA
)
def test_get_kv_cache_spec_kind_unknown_for_mixed_uniform_type_specs():
uniform_mixed_spec = UniformTypeKVCacheSpecs(
block_size=16,
kv_cache_specs={
"layer_1": new_mla_spec(),
"layer_2": SlidingWindowMLASpec(
block_size=16,
num_kv_heads=1,
head_size=576,
dtype=torch.float32,
sliding_window=128,
),
},
)
assert get_kv_cache_spec_kind(uniform_mixed_spec) == KVCacheSpecKind.UNKNOWN
def test_get_kv_cache_spec_sliding_window_reads_windowed_specs():
full_attention_spec = FullAttentionSpec(
block_size=16,
num_kv_heads=1,
head_size=64,
dtype=torch.float32,
)
sliding_window_spec = SlidingWindowSpec(
block_size=16,
num_kv_heads=1,
head_size=64,
dtype=torch.float32,
sliding_window=128,
)
assert get_kv_cache_spec_sliding_window(full_attention_spec) is None
assert get_kv_cache_spec_sliding_window(sliding_window_spec) == 128
def test_get_kv_cache_spec_sliding_window_unwraps_uniform_type_specs():
uniform_window_spec = UniformTypeKVCacheSpecs(
block_size=16,
kv_cache_specs={
"layer_1": SlidingWindowSpec(
block_size=16,
num_kv_heads=1,
head_size=64,
dtype=torch.float32,
sliding_window=128,
),
"layer_2": SlidingWindowSpec(
block_size=16,
num_kv_heads=2,
head_size=64,
dtype=torch.float32,
sliding_window=128,
),
},
)
mixed_window_spec = UniformTypeKVCacheSpecs(
block_size=16,
kv_cache_specs={
"layer_1": SlidingWindowSpec(
block_size=16,
num_kv_heads=1,
head_size=64,
dtype=torch.float32,
sliding_window=128,
),
"layer_2": SlidingWindowSpec(
block_size=16,
num_kv_heads=1,
head_size=64,
dtype=torch.float32,
sliding_window=256,
),
},
)
assert get_kv_cache_spec_sliding_window(uniform_window_spec) == 128
assert get_kv_cache_spec_sliding_window(mixed_window_spec) is None
def test_merge_mla_spec():
kv_cache_specs = [
new_mla_spec(),
+98 -9
View File
@@ -8,6 +8,7 @@ from collections.abc import Callable
import pytest
import torch
import vllm.v1.core.kv_cache_manager as kv_cache_manager
import vllm.v1.core.kv_cache_utils as kv_cache_utils
from vllm.distributed.kv_events import AllBlocksCleared, BlockRemoved, BlockStored
from vllm.lora.request import LoRARequest
@@ -35,6 +36,7 @@ from vllm.v1.kv_cache_interface import (
FullAttentionSpec,
KVCacheConfig,
KVCacheGroupSpec,
KVCacheSpecKind,
MambaSpec,
SlidingWindowSpec,
)
@@ -1933,6 +1935,7 @@ def test_kv_cache_events(blocks_to_cache: int):
== len(manager.block_pool.cached_block_hash_to_block)
)
assert len(block.token_ids) == block.block_size * len(block.block_hashes)
assert block.kv_cache_spec_kind == KVCacheSpecKind.FULL_ATTENTION.value
assert len(manager.block_pool.kv_event_queue) == 0
stored_block_hash = block.block_hashes
@@ -1946,6 +1949,7 @@ def test_kv_cache_events(blocks_to_cache: int):
events = manager.take_events()
for blocks in events[:-1]:
assert isinstance(blocks, BlockRemoved)
assert blocks.block_hashes[0] in stored_block_hash
assert len(events) == blocks_to_cache + 1
assert isinstance(events[-2], BlockRemoved)
@@ -2022,6 +2026,8 @@ def test_null_parent_block_hash():
]
assert event.block_hashes == expected_new_hashes
assert event.group_idx == kv_cache_group_id
assert event.kv_cache_spec_kind is None
assert event.kv_cache_spec_sliding_window is None
# Ensure we didn't accidentally assign a hash to the null block.
assert pool.null_block.block_hash is None
@@ -2095,12 +2101,14 @@ def test_block_stored_event_group_idx(group_id: int):
block_size = 4
num_tokens = block_size * 2
pool = BlockPool(
num_gpu_blocks=5,
manager = KVCacheManager(
make_kv_cache_config_three_types(block_size, num_blocks=5),
max_model_len=8192,
enable_caching=True,
hash_block_size=block_size,
enable_kv_cache_events=True,
hash_block_size=block_size,
)
pool = manager.block_pool
req = make_request(
"req_grp_idx",
@@ -2119,10 +2127,26 @@ def test_block_stored_event_group_idx(group_id: int):
kv_cache_group_id=group_id,
)
events = pool.take_events()
events = manager.take_events()
assert len(events) == 1
assert isinstance(events[0], BlockStored)
assert events[0].group_idx == group_id
assert (
events[0].kv_cache_spec_kind
== [
KVCacheSpecKind.FULL_ATTENTION.value,
KVCacheSpecKind.SLIDING_WINDOW.value,
KVCacheSpecKind.MAMBA.value,
][group_id]
)
assert (
events[0].kv_cache_spec_sliding_window
== [
None,
2 * block_size,
None,
][group_id]
)
def test_block_stored_event_group_idx_multiple_groups():
@@ -2137,13 +2161,38 @@ def test_block_stored_event_group_idx_multiple_groups():
block_size = 4
num_tokens = block_size * 2
# null block + 4 usable (2 per group)
pool = BlockPool(
num_gpu_blocks=5,
manager = KVCacheManager(
KVCacheConfig(
num_blocks=5,
kv_cache_tensors=[],
kv_cache_groups=[
KVCacheGroupSpec(
["layer1"],
FullAttentionSpec(
block_size=block_size,
num_kv_heads=1,
head_size=1,
dtype=torch.float32,
),
),
KVCacheGroupSpec(
["layer2"],
SlidingWindowSpec(
block_size=block_size,
num_kv_heads=1,
head_size=1,
dtype=torch.float32,
sliding_window=128,
),
),
],
),
max_model_len=8192,
enable_caching=True,
hash_block_size=block_size,
enable_kv_cache_events=True,
hash_block_size=block_size,
)
pool = manager.block_pool
req = make_request(
"req_multi_grp",
@@ -2174,12 +2223,52 @@ def test_block_stored_event_group_idx_multiple_groups():
kv_cache_group_id=1,
)
events = pool.take_events()
events = manager.take_events()
assert len(events) == 2
assert isinstance(events[0], BlockStored)
assert events[0].group_idx == 0
assert events[0].kv_cache_spec_kind == KVCacheSpecKind.FULL_ATTENTION.value
assert events[0].kv_cache_spec_sliding_window is None
assert isinstance(events[1], BlockStored)
assert events[1].group_idx == 1
assert events[1].kv_cache_spec_kind == KVCacheSpecKind.SLIDING_WINDOW.value
assert events[1].kv_cache_spec_sliding_window == 128
def test_block_stored_event_group_idx_out_of_bounds(monkeypatch):
"""Out-of-range group_idx events are returned without metadata annotation."""
block_size = 4
manager = KVCacheManager(
make_kv_cache_config(block_size, num_blocks=5),
max_model_len=8192,
enable_caching=True,
enable_kv_cache_events=True,
hash_block_size=block_size,
)
event = BlockStored(
block_hashes=[1],
parent_block_hash=None,
token_ids=list(range(block_size)),
block_size=block_size,
lora_id=None,
medium=None,
lora_name=None,
group_idx=1,
)
manager.block_pool.kv_event_queue.append(event)
warnings = []
def collect_warning(message, *args, **kwargs):
del kwargs
warnings.append(message % args if args else message)
monkeypatch.setattr(kv_cache_manager.logger, "warning", collect_warning)
events = manager.take_events()
assert events == [event]
assert event.kv_cache_spec_kind is None
assert event.kv_cache_spec_sliding_window is None
assert warnings == ["Group index `1` not in KV cache metadata"]
@pytest.mark.parametrize("group_id", [0, 1, 2])