[6/N][KV-Cache Layout Refactor] Standardize KV cache layout (#51718)
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
This commit is contained in:
@@ -92,7 +92,12 @@ def initialize_kv_cache(runner: GPUModelRunner):
|
||||
kv_cache_config = KVCacheConfig(
|
||||
num_blocks=NUM_BLOCKS,
|
||||
kv_cache_tensors=[
|
||||
KVCacheTensor(size=tensor_size, shared_by=["layer.0"]),
|
||||
KVCacheTensor(
|
||||
size=tensor_size,
|
||||
layers=["layer.0"],
|
||||
layer_stride=tensor_size,
|
||||
block_stride=attn_spec.page_size_bytes,
|
||||
),
|
||||
],
|
||||
kv_cache_groups=[
|
||||
KVCacheGroupSpec(layer_names=["layer.0"], kv_cache_spec=attn_spec)
|
||||
@@ -131,6 +136,7 @@ def get_vllm_config():
|
||||
gpu_memory_utilization=0.9,
|
||||
cache_dtype="auto",
|
||||
)
|
||||
cache_config.kv_cache_layout = "LBNHC"
|
||||
parallel_config = ParallelConfig()
|
||||
vllm_config = VllmConfig(
|
||||
model_config=model_config,
|
||||
@@ -839,56 +845,6 @@ def test_update_states_pp_async_multi_request_keeps_rank_state_consistent(
|
||||
)
|
||||
|
||||
|
||||
def test_kv_cache_stride_order(monkeypatch, model_runner):
|
||||
# This test checks if GPUModelRunner initializes correctly when an attention
|
||||
# backend enforces a non-default KV cache stride order.
|
||||
n_heads = model_runner.model_config.get_num_kv_heads(model_runner.parallel_config)
|
||||
head_size = model_runner.model_config.get_head_size()
|
||||
|
||||
# Get the expected shape from the backend's get_kv_cache_shape method
|
||||
# to ensure compatibility with different backends (triton vs flexattention)
|
||||
attn_backend = None
|
||||
for attn_group in model_runner._attn_group_iterator():
|
||||
attn_backend = attn_group.backend
|
||||
break
|
||||
|
||||
assert attn_backend is not None, "No attention backend found"
|
||||
expected_kv_cache_shape = list(
|
||||
attn_backend.get_kv_cache_shape(NUM_BLOCKS, BLOCK_SIZE, n_heads, head_size)
|
||||
)
|
||||
|
||||
# TODO mla test
|
||||
default_stride = tuple(range(len(expected_kv_cache_shape)))
|
||||
non_default_stride = (*default_stride[1:], default_stride[0])
|
||||
# Permutation that gets you back to expected kv shape
|
||||
for test_stride in (non_default_stride, default_stride):
|
||||
|
||||
def rnd_stride_order(
|
||||
include_num_layers_dimension: bool = False, test_stride=test_stride
|
||||
):
|
||||
assert not include_num_layers_dimension
|
||||
return test_stride
|
||||
|
||||
# Patch the attention backend class and re-trigger the KV cache creation
|
||||
for attn_group in model_runner._attn_group_iterator():
|
||||
attn_backend = attn_group.backend
|
||||
monkeypatch.setattr(
|
||||
attn_backend, "get_kv_cache_stride_order", rnd_stride_order
|
||||
)
|
||||
|
||||
model_runner.attn_groups = []
|
||||
model_runner.kv_caches = []
|
||||
model_runner.initialize_kv_cache(model_runner.kv_cache_config)
|
||||
|
||||
# Shape is unchanged, but layout may differ
|
||||
kv_cache_shape = model_runner.kv_caches[0].shape
|
||||
assert list(kv_cache_shape) == expected_kv_cache_shape
|
||||
if default_stride == test_stride:
|
||||
assert all(kv.is_contiguous() for kv in model_runner.kv_caches)
|
||||
else:
|
||||
assert all(not kv.is_contiguous() for kv in model_runner.kv_caches)
|
||||
|
||||
|
||||
def test_update_config(model_runner):
|
||||
# Simple update
|
||||
model_runner.update_config({"load_config": {"load_format": "dummy"}})
|
||||
@@ -1124,21 +1080,22 @@ def test_init_kv_cache_without_kv_sharing(default_vllm_config):
|
||||
vllm_config, [kv_cache_spec], [available_memory]
|
||||
)[0]
|
||||
assert kv_cache_config.num_blocks == num_expected_blocks
|
||||
assert len(kv_cache_config.kv_cache_tensors) == 2
|
||||
assert kv_cache_config.kv_cache_tensors[0].size == available_memory // 2
|
||||
assert kv_cache_config.kv_cache_tensors[1].size == available_memory // 2
|
||||
assert len(kv_cache_config.kv_cache_tensors) == 1
|
||||
assert kv_cache_config.kv_cache_tensors[0].size == available_memory
|
||||
|
||||
max_context_len = estimate_max_model_len(vllm_config, kv_cache_spec, 5 * GiB_bytes)
|
||||
# max context len with KV sharing should be 2x as large as without
|
||||
assert max_context_len == 1310720
|
||||
|
||||
# important: override tensor size to prevent large mem alloc during test
|
||||
# this will only allocate 2 block worth of memory (2 * 32kb)
|
||||
# this will only allocate 1 block worth of memory per layer (2 layers * 32kb)
|
||||
kv_cache_config.num_blocks = 1
|
||||
for kv_cache_tensor in kv_cache_config.kv_cache_tensors:
|
||||
kv_cache_tensor.size = kv_cache_spec[
|
||||
kv_cache_tensor.shared_by[0]
|
||||
].page_size_bytes
|
||||
page_size = kv_cache_spec[kv_cache_tensor.layers[0]].page_size_bytes
|
||||
kv_cache_tensor.size = page_size * len(kv_cache_tensor.layers)
|
||||
# One block per layer: a layer's region is exactly one page.
|
||||
kv_cache_tensor.layer_stride = page_size
|
||||
kv_cache_tensor.block_stride = page_size
|
||||
|
||||
runner.initialize_kv_cache(kv_cache_config)
|
||||
|
||||
@@ -1207,7 +1164,9 @@ def test_init_kv_cache_with_kv_sharing_valid(default_vllm_config):
|
||||
# important: override tensor size to prevent large mem alloc during test
|
||||
# this will only allocate 1 block worth of memory (32kb)
|
||||
kv_cache_config.num_blocks = 1
|
||||
kv_cache_config.kv_cache_tensors[0].size = kv_cache_spec[layer_0].page_size_bytes
|
||||
page_size = kv_cache_spec[layer_0].page_size_bytes
|
||||
kv_cache_config.kv_cache_tensors[0].size = page_size
|
||||
kv_cache_config.kv_cache_tensors[0].layer_stride = page_size
|
||||
|
||||
runner.initialize_kv_cache(kv_cache_config)
|
||||
kv_cache_config_after_init = runner.kv_cache_config
|
||||
@@ -1232,7 +1191,7 @@ def test_hybrid_attention_mamba_tensor_shapes():
|
||||
"""
|
||||
The GPU model runner creates different views into the
|
||||
KVCacheTensors for the attention and mamba layers
|
||||
(via _reshape_kv_cache_tensors function). This test verifies
|
||||
(via _allocate_kv_caches). This test verifies
|
||||
that the views are compatible: writing a mamba block
|
||||
will not corrupt an attention block and vice versa
|
||||
"""
|
||||
@@ -1270,6 +1229,7 @@ def test_hybrid_attention_mamba_tensor_shapes():
|
||||
gpu_memory_utilization=0.9,
|
||||
cache_dtype="auto",
|
||||
)
|
||||
cache_config.kv_cache_layout = "LBNHC"
|
||||
parallel_config = ParallelConfig()
|
||||
attention_config = AttentionConfig(backend=AttentionBackendEnum.FLASHINFER)
|
||||
vllm_config = VllmConfig(
|
||||
@@ -1561,7 +1521,12 @@ def test_hybrid_cache_integration(default_vllm_config, dist_init):
|
||||
kv_cache_config = KVCacheConfig(
|
||||
num_blocks=NUM_BLOCKS,
|
||||
kv_cache_tensors=[
|
||||
KVCacheTensor(size=tensor_size, shared_by=["layer.0"]),
|
||||
KVCacheTensor(
|
||||
size=tensor_size,
|
||||
layers=["layer.0"],
|
||||
layer_stride=tensor_size,
|
||||
block_stride=attn_spec.page_size_bytes,
|
||||
),
|
||||
],
|
||||
kv_cache_groups=[
|
||||
KVCacheGroupSpec(layer_names=["layer.0"], kv_cache_spec=attn_spec)
|
||||
@@ -1726,6 +1691,7 @@ def test_mamba_cache_raises_when_max_num_seqs_exceeds_blocks():
|
||||
gpu_memory_utilization=0.9,
|
||||
cache_dtype="auto",
|
||||
)
|
||||
cache_config.kv_cache_layout = "LBNHC"
|
||||
parallel_config = ParallelConfig()
|
||||
attention_config = AttentionConfig(backend=AttentionBackendEnum.FLASHINFER)
|
||||
vllm_config = VllmConfig(
|
||||
|
||||
Reference in New Issue
Block a user