[6/N][KV-Cache Layout Refactor] Standardize KV cache layout (#51718)

Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
This commit is contained in:
Lucas Wilkinson
2026-08-21 17:47:49 -07:00
committed by GitHub
parent 9ff7041b55
commit 8bdc70ec7b
150 changed files with 4006 additions and 5815 deletions
+27 -61
View File
@@ -92,7 +92,12 @@ def initialize_kv_cache(runner: GPUModelRunner):
kv_cache_config = KVCacheConfig(
num_blocks=NUM_BLOCKS,
kv_cache_tensors=[
KVCacheTensor(size=tensor_size, shared_by=["layer.0"]),
KVCacheTensor(
size=tensor_size,
layers=["layer.0"],
layer_stride=tensor_size,
block_stride=attn_spec.page_size_bytes,
),
],
kv_cache_groups=[
KVCacheGroupSpec(layer_names=["layer.0"], kv_cache_spec=attn_spec)
@@ -131,6 +136,7 @@ def get_vllm_config():
gpu_memory_utilization=0.9,
cache_dtype="auto",
)
cache_config.kv_cache_layout = "LBNHC"
parallel_config = ParallelConfig()
vllm_config = VllmConfig(
model_config=model_config,
@@ -839,56 +845,6 @@ def test_update_states_pp_async_multi_request_keeps_rank_state_consistent(
)
def test_kv_cache_stride_order(monkeypatch, model_runner):
# This test checks if GPUModelRunner initializes correctly when an attention
# backend enforces a non-default KV cache stride order.
n_heads = model_runner.model_config.get_num_kv_heads(model_runner.parallel_config)
head_size = model_runner.model_config.get_head_size()
# Get the expected shape from the backend's get_kv_cache_shape method
# to ensure compatibility with different backends (triton vs flexattention)
attn_backend = None
for attn_group in model_runner._attn_group_iterator():
attn_backend = attn_group.backend
break
assert attn_backend is not None, "No attention backend found"
expected_kv_cache_shape = list(
attn_backend.get_kv_cache_shape(NUM_BLOCKS, BLOCK_SIZE, n_heads, head_size)
)
# TODO mla test
default_stride = tuple(range(len(expected_kv_cache_shape)))
non_default_stride = (*default_stride[1:], default_stride[0])
# Permutation that gets you back to expected kv shape
for test_stride in (non_default_stride, default_stride):
def rnd_stride_order(
include_num_layers_dimension: bool = False, test_stride=test_stride
):
assert not include_num_layers_dimension
return test_stride
# Patch the attention backend class and re-trigger the KV cache creation
for attn_group in model_runner._attn_group_iterator():
attn_backend = attn_group.backend
monkeypatch.setattr(
attn_backend, "get_kv_cache_stride_order", rnd_stride_order
)
model_runner.attn_groups = []
model_runner.kv_caches = []
model_runner.initialize_kv_cache(model_runner.kv_cache_config)
# Shape is unchanged, but layout may differ
kv_cache_shape = model_runner.kv_caches[0].shape
assert list(kv_cache_shape) == expected_kv_cache_shape
if default_stride == test_stride:
assert all(kv.is_contiguous() for kv in model_runner.kv_caches)
else:
assert all(not kv.is_contiguous() for kv in model_runner.kv_caches)
def test_update_config(model_runner):
# Simple update
model_runner.update_config({"load_config": {"load_format": "dummy"}})
@@ -1124,21 +1080,22 @@ def test_init_kv_cache_without_kv_sharing(default_vllm_config):
vllm_config, [kv_cache_spec], [available_memory]
)[0]
assert kv_cache_config.num_blocks == num_expected_blocks
assert len(kv_cache_config.kv_cache_tensors) == 2
assert kv_cache_config.kv_cache_tensors[0].size == available_memory // 2
assert kv_cache_config.kv_cache_tensors[1].size == available_memory // 2
assert len(kv_cache_config.kv_cache_tensors) == 1
assert kv_cache_config.kv_cache_tensors[0].size == available_memory
max_context_len = estimate_max_model_len(vllm_config, kv_cache_spec, 5 * GiB_bytes)
# max context len with KV sharing should be 2x as large as without
assert max_context_len == 1310720
# important: override tensor size to prevent large mem alloc during test
# this will only allocate 2 block worth of memory (2 * 32kb)
# this will only allocate 1 block worth of memory per layer (2 layers * 32kb)
kv_cache_config.num_blocks = 1
for kv_cache_tensor in kv_cache_config.kv_cache_tensors:
kv_cache_tensor.size = kv_cache_spec[
kv_cache_tensor.shared_by[0]
].page_size_bytes
page_size = kv_cache_spec[kv_cache_tensor.layers[0]].page_size_bytes
kv_cache_tensor.size = page_size * len(kv_cache_tensor.layers)
# One block per layer: a layer's region is exactly one page.
kv_cache_tensor.layer_stride = page_size
kv_cache_tensor.block_stride = page_size
runner.initialize_kv_cache(kv_cache_config)
@@ -1207,7 +1164,9 @@ def test_init_kv_cache_with_kv_sharing_valid(default_vllm_config):
# important: override tensor size to prevent large mem alloc during test
# this will only allocate 1 block worth of memory (32kb)
kv_cache_config.num_blocks = 1
kv_cache_config.kv_cache_tensors[0].size = kv_cache_spec[layer_0].page_size_bytes
page_size = kv_cache_spec[layer_0].page_size_bytes
kv_cache_config.kv_cache_tensors[0].size = page_size
kv_cache_config.kv_cache_tensors[0].layer_stride = page_size
runner.initialize_kv_cache(kv_cache_config)
kv_cache_config_after_init = runner.kv_cache_config
@@ -1232,7 +1191,7 @@ def test_hybrid_attention_mamba_tensor_shapes():
"""
The GPU model runner creates different views into the
KVCacheTensors for the attention and mamba layers
(via _reshape_kv_cache_tensors function). This test verifies
(via _allocate_kv_caches). This test verifies
that the views are compatible: writing a mamba block
will not corrupt an attention block and vice versa
"""
@@ -1270,6 +1229,7 @@ def test_hybrid_attention_mamba_tensor_shapes():
gpu_memory_utilization=0.9,
cache_dtype="auto",
)
cache_config.kv_cache_layout = "LBNHC"
parallel_config = ParallelConfig()
attention_config = AttentionConfig(backend=AttentionBackendEnum.FLASHINFER)
vllm_config = VllmConfig(
@@ -1561,7 +1521,12 @@ def test_hybrid_cache_integration(default_vllm_config, dist_init):
kv_cache_config = KVCacheConfig(
num_blocks=NUM_BLOCKS,
kv_cache_tensors=[
KVCacheTensor(size=tensor_size, shared_by=["layer.0"]),
KVCacheTensor(
size=tensor_size,
layers=["layer.0"],
layer_stride=tensor_size,
block_stride=attn_spec.page_size_bytes,
),
],
kv_cache_groups=[
KVCacheGroupSpec(layer_names=["layer.0"], kv_cache_spec=attn_spec)
@@ -1726,6 +1691,7 @@ def test_mamba_cache_raises_when_max_num_seqs_exceeds_blocks():
gpu_memory_utilization=0.9,
cache_dtype="auto",
)
cache_config.kv_cache_layout = "LBNHC"
parallel_config = ParallelConfig()
attention_config = AttentionConfig(backend=AttentionBackendEnum.FLASHINFER)
vllm_config = VllmConfig(