[6/N][KV-Cache Layout Refactor] Standardize KV cache layout (#51718)
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
This commit is contained in:
+191
-306
@@ -1,48 +1,33 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Padded-page handling in create_kv_cache_views.
|
||||
|
||||
Guards that a page_size_padded spec strides the block dimension by the padded page
|
||||
while keeping per-block content compact, so padding bytes at the end of each page are
|
||||
never addressed by the logical view.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from tests.v1.attention.utils import dense_kv_cache_views
|
||||
from vllm.v1.core.kv_cache_utils import KVCacheBlockCopy
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
FullAttentionSpec,
|
||||
KVCacheConfig,
|
||||
KVCacheGroupSpec,
|
||||
KVCacheLayout,
|
||||
KVCacheTensor,
|
||||
KVQuantMode,
|
||||
MambaSpec,
|
||||
MLAAttentionSpec,
|
||||
compute_layout_strides,
|
||||
)
|
||||
from vllm.v1.worker.utils import (
|
||||
allocate_kv_cache,
|
||||
copy_kv_cache_blocks_inplace,
|
||||
)
|
||||
from vllm.v1.worker.gpu.attn_utils import _reshape_kv_cache
|
||||
from vllm.v1.worker.utils import AttentionGroup
|
||||
|
||||
|
||||
class FakeFlashAttentionBackend:
|
||||
@staticmethod
|
||||
def get_kv_cache_shape(
|
||||
num_blocks: int,
|
||||
block_size: int,
|
||||
num_kv_heads: int,
|
||||
head_size: int,
|
||||
cache_dtype_str: str = "auto",
|
||||
) -> tuple[int, ...]:
|
||||
return (num_blocks, 2, block_size, num_kv_heads, head_size)
|
||||
|
||||
@staticmethod
|
||||
def get_kv_cache_stride_order(
|
||||
include_num_layers_dimension: bool = False,
|
||||
) -> tuple[int, ...]:
|
||||
assert not include_num_layers_dimension
|
||||
return (0, 1, 2, 3, 4)
|
||||
|
||||
|
||||
class FakeHNDFlashAttentionBackend(FakeFlashAttentionBackend):
|
||||
@staticmethod
|
||||
def get_kv_cache_stride_order(
|
||||
include_num_layers_dimension: bool = False,
|
||||
) -> tuple[int, ...]:
|
||||
assert not include_num_layers_dimension
|
||||
return (0, 1, 3, 2, 4)
|
||||
|
||||
|
||||
def test_reshape_padded_flash_attention_kv_cache_strides_by_page():
|
||||
def test_reshape_padded_kv_cache_strides_by_padded_page():
|
||||
num_blocks = 3
|
||||
spec = FullAttentionSpec(
|
||||
block_size=16,
|
||||
@@ -53,292 +38,192 @@ def test_reshape_padded_flash_attention_kv_cache_strides_by_page():
|
||||
)
|
||||
assert spec.real_page_size_bytes == 256
|
||||
|
||||
raw_tensors = {
|
||||
"layer": torch.zeros(spec.page_size_bytes * num_blocks, dtype=torch.int8)
|
||||
}
|
||||
attn_groups = [
|
||||
AttentionGroup(
|
||||
backend=FakeFlashAttentionBackend,
|
||||
layer_names=["layer"],
|
||||
kv_cache_spec=spec,
|
||||
kv_cache_group_id=0,
|
||||
)
|
||||
]
|
||||
raw = torch.zeros(spec.page_size_bytes * num_blocks, dtype=torch.int8)
|
||||
(kv_cache,) = dense_kv_cache_views(raw, spec, num_blocks, 1, KVCacheLayout.LBHNC)
|
||||
|
||||
kv_cache = _reshape_kv_cache(
|
||||
attn_groups,
|
||||
raw_tensors,
|
||||
"auto",
|
||||
[spec.block_size],
|
||||
{},
|
||||
)["layer"]
|
||||
|
||||
assert kv_cache.shape == (num_blocks, 2, 16, 1, 2)
|
||||
assert kv_cache.stride(0) == spec.page_size_bytes // 4
|
||||
assert kv_cache.stride(1) == spec.real_page_size_bytes // 2 // 4
|
||||
assert kv_cache[1, 0].storage_offset() == spec.page_size_bytes // 4
|
||||
assert (
|
||||
kv_cache[1, 1].storage_offset()
|
||||
== (spec.page_size_bytes + spec.real_page_size_bytes // 2) // 4
|
||||
)
|
||||
elem_size = 4 # float32
|
||||
# Content dim packs K and V: 2 * head_size.
|
||||
assert kv_cache.shape == (num_blocks, 1, 16, 2 * spec.head_size)
|
||||
assert kv_cache.dtype == spec.dtype
|
||||
assert kv_cache.stride(0) == spec.page_size_padded // elem_size
|
||||
assert kv_cache[1].storage_offset() == spec.page_size_padded // elem_size
|
||||
# Within one block the (unpadded) content stays compact.
|
||||
assert kv_cache[0].is_contiguous()
|
||||
|
||||
|
||||
def test_reshape_padded_hnd_flash_attention_kv_cache_strides_by_page():
|
||||
num_blocks = 3
|
||||
spec = FullAttentionSpec(
|
||||
block_size=16,
|
||||
num_kv_heads=3,
|
||||
head_size=2,
|
||||
dtype=torch.float32,
|
||||
page_size_padded=1024,
|
||||
)
|
||||
assert spec.real_page_size_bytes == 768
|
||||
|
||||
raw_tensors = {
|
||||
"layer": torch.zeros(spec.page_size_bytes * num_blocks, dtype=torch.int8)
|
||||
}
|
||||
attn_groups = [
|
||||
AttentionGroup(
|
||||
backend=FakeHNDFlashAttentionBackend,
|
||||
layer_names=["layer"],
|
||||
kv_cache_spec=spec,
|
||||
kv_cache_group_id=0,
|
||||
)
|
||||
]
|
||||
|
||||
kv_cache = _reshape_kv_cache(
|
||||
attn_groups,
|
||||
raw_tensors,
|
||||
"auto",
|
||||
[spec.block_size],
|
||||
{},
|
||||
)["layer"]
|
||||
|
||||
assert kv_cache.shape == (num_blocks, 2, 16, 3, 2)
|
||||
assert kv_cache.stride(0) == spec.page_size_bytes // 4
|
||||
assert kv_cache.stride(1) == spec.real_page_size_bytes // 2 // 4
|
||||
assert kv_cache.stride(2) == 2
|
||||
assert kv_cache.stride(3) == spec.block_size * spec.head_size
|
||||
assert kv_cache[1, 0].storage_offset() == spec.page_size_bytes // 4
|
||||
assert (
|
||||
kv_cache[1, 1].storage_offset()
|
||||
== (spec.page_size_bytes + spec.real_page_size_bytes // 2) // 4
|
||||
)
|
||||
assert (
|
||||
kv_cache[1, 1, 3, 2].storage_offset()
|
||||
== (
|
||||
spec.page_size_bytes
|
||||
+ spec.real_page_size_bytes // 2
|
||||
+ 3 * spec.head_size * 4
|
||||
+ 2 * spec.block_size * spec.head_size * 4
|
||||
)
|
||||
// 4
|
||||
)
|
||||
|
||||
|
||||
class FakeDiffKVBackend:
|
||||
@staticmethod
|
||||
def get_kv_cache_shape(
|
||||
num_blocks: int,
|
||||
block_size: int,
|
||||
num_kv_heads: int,
|
||||
head_size: int,
|
||||
cache_dtype_str: str = "auto",
|
||||
) -> tuple[int, ...]:
|
||||
return (num_blocks, block_size, num_kv_heads, head_size * 2)
|
||||
|
||||
@staticmethod
|
||||
def get_kv_cache_stride_order(
|
||||
include_num_layers_dimension: bool = False,
|
||||
) -> tuple[int, ...]:
|
||||
assert not include_num_layers_dimension
|
||||
return (0, 1, 2, 3)
|
||||
|
||||
|
||||
def test_reshape_padded_diff_kv_cache_does_not_infer_kv_dim():
|
||||
num_blocks = 3
|
||||
spec = FullAttentionSpec(
|
||||
block_size=16,
|
||||
@pytest.mark.parametrize(
|
||||
("kernel_block_sizes", "expected_num_blocks", "expected_num_states"),
|
||||
[
|
||||
(None, 4, 64),
|
||||
([256], 4, 64),
|
||||
([64], 16, 16),
|
||||
],
|
||||
)
|
||||
def test_allocate_compressed_mla_cache(
|
||||
kernel_block_sizes: list[int] | None,
|
||||
expected_num_blocks: int,
|
||||
expected_num_states: int,
|
||||
):
|
||||
spec = MLAAttentionSpec(
|
||||
block_size=256,
|
||||
num_kv_heads=1,
|
||||
head_size=2,
|
||||
dtype=torch.float32,
|
||||
page_size_padded=384,
|
||||
head_size=128,
|
||||
dtype=torch.bfloat16,
|
||||
tokens_per_state=4,
|
||||
)
|
||||
|
||||
raw_tensors = {
|
||||
"layer": torch.zeros(spec.page_size_bytes * num_blocks, dtype=torch.int8)
|
||||
}
|
||||
attn_groups = [
|
||||
AttentionGroup(
|
||||
backend=FakeDiffKVBackend,
|
||||
layer_names=["layer"],
|
||||
kv_cache_spec=spec,
|
||||
kv_cache_group_id=0,
|
||||
)
|
||||
]
|
||||
|
||||
kv_cache = _reshape_kv_cache(
|
||||
attn_groups,
|
||||
raw_tensors,
|
||||
"auto",
|
||||
[spec.block_size],
|
||||
{},
|
||||
)["layer"]
|
||||
|
||||
assert kv_cache.shape == (num_blocks, 16, 1, 4)
|
||||
assert kv_cache.stride(0) == spec.page_size_bytes // 4
|
||||
assert kv_cache.stride(1) == 4
|
||||
|
||||
|
||||
class FakePerTokenScaleBackend:
|
||||
@staticmethod
|
||||
def get_kv_cache_shape(
|
||||
num_blocks: int,
|
||||
block_size: int,
|
||||
num_kv_heads: int,
|
||||
head_size: int,
|
||||
cache_dtype_str: str = "auto",
|
||||
) -> tuple[int, ...]:
|
||||
return (num_blocks, 2, block_size, num_kv_heads, head_size + 4)
|
||||
|
||||
@staticmethod
|
||||
def get_kv_cache_stride_order(
|
||||
include_num_layers_dimension: bool = False,
|
||||
) -> tuple[int, ...]:
|
||||
assert not include_num_layers_dimension
|
||||
return (0, 1, 2, 3, 4)
|
||||
|
||||
|
||||
def test_reshape_padded_quantized_kv_cache_preserves_scale_stride():
|
||||
num_blocks = 3
|
||||
spec = FullAttentionSpec(
|
||||
block_size=16,
|
||||
num_kv_heads=1,
|
||||
head_size=4,
|
||||
dtype=torch.int8,
|
||||
kv_quant_mode=KVQuantMode.INT8_PER_TOKEN_HEAD,
|
||||
page_size_padded=384,
|
||||
)
|
||||
assert spec.real_page_size_bytes == 128
|
||||
assert spec.page_size_bytes == 384
|
||||
|
||||
raw_tensors = {
|
||||
"layer": torch.zeros(spec.page_size_bytes * num_blocks, dtype=torch.int8)
|
||||
}
|
||||
attn_groups = [
|
||||
AttentionGroup(
|
||||
backend=FakePerTokenScaleBackend,
|
||||
layer_names=["layer"],
|
||||
kv_cache_spec=spec,
|
||||
kv_cache_group_id=0,
|
||||
)
|
||||
]
|
||||
|
||||
kv_cache = _reshape_kv_cache(
|
||||
attn_groups,
|
||||
raw_tensors,
|
||||
"int8_per_token_head",
|
||||
[spec.block_size],
|
||||
{},
|
||||
)["layer"]
|
||||
|
||||
assert kv_cache.shape == (num_blocks, 2, 16, 1, 8)
|
||||
assert kv_cache.stride(0) == spec.page_size_bytes
|
||||
assert kv_cache.stride(1) == 16 * 1 * 8
|
||||
assert kv_cache[1, 1].storage_offset() == spec.page_size_bytes + 16 * 1 * 8
|
||||
|
||||
|
||||
class FakeKVFirstBackend:
|
||||
"""ROCm-style backend that puts K and V ahead of the block dim."""
|
||||
|
||||
@staticmethod
|
||||
def get_kv_cache_shape(
|
||||
num_blocks: int,
|
||||
block_size: int,
|
||||
num_kv_heads: int,
|
||||
head_size: int,
|
||||
cache_dtype_str: str = "auto",
|
||||
) -> tuple[int, ...]:
|
||||
return (2, num_blocks, block_size, num_kv_heads, head_size)
|
||||
|
||||
@staticmethod
|
||||
def get_kv_cache_block_dim(
|
||||
block_size: int,
|
||||
num_kv_heads: int,
|
||||
head_size: int,
|
||||
cache_dtype_str: str = "auto",
|
||||
) -> int:
|
||||
return 1
|
||||
|
||||
|
||||
def _kv_first_setup(shared_by: list[str]):
|
||||
num_blocks = 3
|
||||
attn_spec = FullAttentionSpec(
|
||||
block_size=16,
|
||||
num_kv_heads=1,
|
||||
head_size=2,
|
||||
dtype=torch.float32,
|
||||
)
|
||||
mamba_spec = MambaSpec(
|
||||
block_size=16,
|
||||
shapes=((64,),),
|
||||
dtypes=(torch.float32,),
|
||||
)
|
||||
assert attn_spec.page_size_bytes == mamba_spec.page_size_bytes == 256
|
||||
|
||||
raw_tensor = torch.zeros(attn_spec.page_size_bytes * num_blocks, dtype=torch.int8)
|
||||
raw_tensors = {name: raw_tensor for name in shared_by}
|
||||
attn_groups = [
|
||||
AttentionGroup(
|
||||
backend=FakeKVFirstBackend,
|
||||
layer_names=["attn"],
|
||||
kv_cache_spec=attn_spec,
|
||||
kv_cache_group_id=0,
|
||||
)
|
||||
]
|
||||
if "mamba" in shared_by:
|
||||
attn_groups.append(
|
||||
AttentionGroup(
|
||||
backend=FakeKVFirstBackend,
|
||||
layer_names=["mamba"],
|
||||
kv_cache_spec=mamba_spec,
|
||||
kv_cache_group_id=1,
|
||||
)
|
||||
)
|
||||
kv_cache_config = KVCacheConfig(
|
||||
num_blocks=num_blocks,
|
||||
num_pages = 4
|
||||
config = KVCacheConfig(
|
||||
num_blocks=num_pages,
|
||||
kv_cache_tensors=[
|
||||
KVCacheTensor(size=raw_tensor.numel(), shared_by=list(shared_by))
|
||||
KVCacheTensor(
|
||||
size=num_pages * spec.page_size_bytes,
|
||||
layers=["layer.0"],
|
||||
layer_stride=num_pages * spec.page_size_bytes,
|
||||
block_stride=spec.page_size_bytes,
|
||||
)
|
||||
],
|
||||
kv_cache_groups=[],
|
||||
kv_cache_groups=[KVCacheGroupSpec(["layer.0"], spec)],
|
||||
)
|
||||
|
||||
kv_caches = _reshape_kv_cache(
|
||||
attn_groups,
|
||||
raw_tensors,
|
||||
"auto",
|
||||
[attn_spec.block_size] * len(attn_groups),
|
||||
{},
|
||||
kv_cache_config,
|
||||
caches = allocate_kv_cache(
|
||||
config, torch.device("cpu"), KVCacheLayout.LBHNC, kernel_block_sizes
|
||||
)
|
||||
return num_blocks, kv_caches["attn"]
|
||||
|
||||
assert caches["layer.0"].shape == (expected_num_blocks, 1, expected_num_states, 128)
|
||||
|
||||
|
||||
def test_reshape_kv_first_kv_cache_pages_blocks_when_shared_with_mamba():
|
||||
num_blocks, kv_cache = _kv_first_setup(["attn", "mamba"])
|
||||
@pytest.mark.parametrize("layout", list(KVCacheLayout))
|
||||
def test_copy_kv_cache_blocks_shared_storage(layout: KVCacheLayout):
|
||||
num_blocks = 4
|
||||
num_layers = 2
|
||||
spec = FullAttentionSpec(
|
||||
block_size=2,
|
||||
num_kv_heads=2,
|
||||
head_size=2,
|
||||
dtype=torch.float32,
|
||||
)
|
||||
raw = torch.zeros(num_blocks * num_layers * spec.page_size_bytes, dtype=torch.int8)
|
||||
caches = dense_kv_cache_views(raw, spec, num_blocks, num_layers, layout)
|
||||
|
||||
assert kv_cache.shape == (2, num_blocks, 16, 1, 2)
|
||||
# Block b has to own page b, so its K and V sit side by side within it.
|
||||
page = 16 * 1 * 2 * 2
|
||||
for block in range(num_blocks):
|
||||
assert kv_cache[0, block].storage_offset() == block * page
|
||||
assert kv_cache[1, block].storage_offset() == block * page + page // 2
|
||||
for layer_idx, cache in enumerate(caches):
|
||||
for block_idx in range(num_blocks):
|
||||
cache[block_idx].fill_(10 * layer_idx + block_idx)
|
||||
|
||||
expected = [[cache[i].clone() for i in range(num_blocks)] for cache in caches]
|
||||
copies = [KVCacheBlockCopy(src_block_id=0, dst_block_id=2)]
|
||||
|
||||
copy_kv_cache_blocks_inplace(caches, num_blocks, copies)
|
||||
|
||||
for layer_idx, cache in enumerate(caches):
|
||||
torch.testing.assert_close(cache[2], expected[layer_idx][0])
|
||||
torch.testing.assert_close(cache[1], expected[layer_idx][1])
|
||||
|
||||
|
||||
def test_reshape_kv_first_kv_cache_keeps_layout_without_mamba():
|
||||
num_blocks, kv_cache = _kv_first_setup(["attn"])
|
||||
def test_fixed_block_stride_propagates_outward_in_lhbnc():
|
||||
num_blocks = 3
|
||||
num_layers = 2
|
||||
spec = FullAttentionSpec(
|
||||
block_size=2,
|
||||
num_kv_heads=2,
|
||||
head_size=2,
|
||||
dtype=torch.float32,
|
||||
)
|
||||
natural = compute_layout_strides(spec, num_blocks, num_layers, KVCacheLayout.LHBNC)
|
||||
block_stride = natural[1] + 8
|
||||
|
||||
assert kv_cache.shape == (2, num_blocks, 16, 1, 2)
|
||||
# Nothing else indexes this allocation by page, so K and V stay split into
|
||||
# one contiguous half each.
|
||||
assert kv_cache[1, 0].storage_offset() == num_blocks * 16 * 1 * 2
|
||||
strides = compute_layout_strides(
|
||||
spec,
|
||||
num_blocks,
|
||||
num_layers,
|
||||
KVCacheLayout.LHBNC,
|
||||
fixed_strides=(None, block_stride, None, None, None),
|
||||
)
|
||||
|
||||
assert strides[1] == block_stride
|
||||
assert strides[2] == block_stride * num_blocks
|
||||
assert strides[0] == strides[2] * spec.num_heads
|
||||
|
||||
|
||||
def test_copy_kv_cache_blocks_separate_head_groups():
|
||||
# LHBNC stores each head group separately, so a block's bytes are scattered
|
||||
# across L*H regions.
|
||||
layout = KVCacheLayout.LHBNC
|
||||
num_blocks = 4
|
||||
num_layers = 2
|
||||
spec = FullAttentionSpec(
|
||||
block_size=2,
|
||||
num_kv_heads=2,
|
||||
head_size=2,
|
||||
dtype=torch.float32,
|
||||
num_head_slots=2,
|
||||
state_content_bytes=2 * 2 * 4,
|
||||
)
|
||||
raw = torch.zeros(num_blocks * num_layers * spec.page_size_bytes, dtype=torch.int8)
|
||||
caches = dense_kv_cache_views(raw, spec, num_blocks, num_layers, layout)
|
||||
|
||||
for layer_idx, cache in enumerate(caches):
|
||||
for block_idx in range(num_blocks):
|
||||
for head_idx in range(cache.shape[1]):
|
||||
cache[block_idx, head_idx].fill_(
|
||||
100 * layer_idx + 10 * head_idx + block_idx
|
||||
)
|
||||
|
||||
expected = [[cache[i].clone() for i in range(num_blocks)] for cache in caches]
|
||||
copy_kv_cache_blocks_inplace(
|
||||
caches,
|
||||
num_blocks,
|
||||
[KVCacheBlockCopy(src_block_id=0, dst_block_id=2)],
|
||||
)
|
||||
|
||||
for layer_idx, cache in enumerate(caches):
|
||||
torch.testing.assert_close(cache[2], expected[layer_idx][0])
|
||||
torch.testing.assert_close(cache[1], expected[layer_idx][1])
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"layout,num_layers",
|
||||
[
|
||||
(KVCacheLayout.LBHNC, 2),
|
||||
# Splitting needs a manager block to be one dense page, which a
|
||||
# block-outermost layout only gives when the block holds one layer.
|
||||
(KVCacheLayout.BLHNC, 1),
|
||||
],
|
||||
)
|
||||
def test_copy_kv_cache_blocks_with_virtual_block_splitting(
|
||||
layout: KVCacheLayout, num_layers: int
|
||||
):
|
||||
num_blocks = 4
|
||||
physical_per_logical = 2
|
||||
spec = FullAttentionSpec(
|
||||
block_size=4,
|
||||
num_kv_heads=1,
|
||||
head_size=2,
|
||||
dtype=torch.float32,
|
||||
)
|
||||
raw = torch.zeros(num_blocks * num_layers * spec.page_size_bytes, dtype=torch.int8)
|
||||
caches = dense_kv_cache_views(
|
||||
raw,
|
||||
spec,
|
||||
num_blocks,
|
||||
num_layers,
|
||||
layout,
|
||||
kernel_block_size=spec.block_size // physical_per_logical,
|
||||
)
|
||||
|
||||
for layer_idx, cache in enumerate(caches):
|
||||
for block_idx in range(cache.shape[0]):
|
||||
cache[block_idx].fill_(100 * layer_idx + block_idx)
|
||||
expected = [[cache[i].clone() for i in range(cache.shape[0])] for cache in caches]
|
||||
|
||||
copy_kv_cache_blocks_inplace(
|
||||
caches,
|
||||
num_blocks,
|
||||
[KVCacheBlockCopy(src_block_id=0, dst_block_id=2)],
|
||||
)
|
||||
|
||||
dst_start = 2 * physical_per_logical
|
||||
for layer_idx, cache in enumerate(caches):
|
||||
for physical_idx in range(physical_per_logical):
|
||||
torch.testing.assert_close(
|
||||
cache[dst_start + physical_idx], expected[layer_idx][physical_idx]
|
||||
)
|
||||
|
||||
@@ -1,35 +1,14 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""CPU regression for the packed-DSV4 KV zeroer geometry.
|
||||
"""CPU regression for the packed-DSV4 KV zeroer geometry (upstream #50276).
|
||||
|
||||
Constructor-crossing regression for the packed-KV zeroing stride fix (upstream
|
||||
PR #50276): every number below comes from instantiating the REAL production
|
||||
code (no arithmetic is reimplemented here):
|
||||
Four DSV4 fp8_ds_mla layers share one cross-layer allocation: each block row packs
|
||||
the four alignment-padded pages side by side (layout BLHNC). The zeroer must step by
|
||||
the full packed row per block while zeroing only each layer's meaningful page — never
|
||||
the alignment padding, never an adjacent layer's page.
|
||||
|
||||
- ``vllm.v1.core.kv_cache_utils._get_packed_kv_cache_layout`` (packed
|
||||
offsets / block stride)
|
||||
- ``vllm.v1.worker.gpu.attn_utils._reshape_attention_kv_cache``
|
||||
(per-layer column views)
|
||||
- ``vllm.models.deepseek_v4.sparse_mla.DeepseekV4FlashMLABackend``
|
||||
(``get_kv_cache_shape`` / ``get_kv_cache_block_dim``)
|
||||
- ``vllm.v1.worker.utils.AttentionGroup`` / ``KVBlockZeroer``
|
||||
(``KVBlockZeroer.__init__`` precomputes the segment tables)
|
||||
|
||||
CPU only: no Triton launch, no CUDA tensor, no CUDA call is made. The test
|
||||
asserts the packed-DSV4 constructor metadata:
|
||||
|
||||
* block stride = the full packed row (4 aligned pages)
|
||||
* per-layer zero = the meaningful page (the spec's unpadded
|
||||
``real_page_size_bytes``), so adjacent packed layers are
|
||||
never wiped
|
||||
* 4 packed layer views; block 99 of the highest-offset layer is the last
|
||||
written region and stays within the packed backing.
|
||||
|
||||
The pre-fix constructor exposed a five-field ``_meta`` with no block stride, so
|
||||
its zero width equals the FULL packed row; the extraction below records that
|
||||
legacy defect and then asserts the fixed semantics, so failure on the old code
|
||||
is a semantic one (wrong zero width / out-of-bounds block 99), not a
|
||||
schema-shape accident.
|
||||
CPU only: the zeroer's ``__init__`` precomputes the segment tables; no kernel is
|
||||
launched.
|
||||
"""
|
||||
|
||||
from types import SimpleNamespace
|
||||
@@ -37,160 +16,154 @@ from types import SimpleNamespace
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.models.deepseek_v4.sparse_mla import DeepseekV4FlashMLABackend
|
||||
from vllm.v1.core.kv_cache_utils import _get_packed_kv_cache_layout
|
||||
from vllm.v1.kv_cache_interface import MLAAttentionSpec, UniformTypeKVCacheSpecs
|
||||
from vllm.v1.worker.gpu.attn_utils import _reshape_attention_kv_cache
|
||||
from vllm.v1.worker.utils import AttentionGroup, KVBlockZeroer
|
||||
from tests.v1.attention.utils import dense_kv_cache_views
|
||||
from vllm.v1.kv_cache_interface import KVCacheLayout, MLAAttentionSpec
|
||||
from vllm.v1.worker.utils import (
|
||||
AttentionGroup,
|
||||
KVBlockZeroer,
|
||||
allocate_kv_cache,
|
||||
)
|
||||
|
||||
pytestmark = pytest.mark.cpu_test
|
||||
|
||||
NUM_BLOCKS = 100
|
||||
BLOCK_SIZE_CFG = 256 # vllm_config.cache_config.block_size for DSV4 (group 0)
|
||||
COMPRESS = 4 # main-MLA compress_ratio
|
||||
STORAGE_BS = BLOCK_SIZE_CFG // COMPRESS # 64
|
||||
KERNEL_BS = 256 # DeepseekV4FlashMLABackend supports [256]
|
||||
LAST_BLOCK = NUM_BLOCKS - 1 # block 99
|
||||
NUM_LAYERS = 4
|
||||
ALIGNMENT = 576
|
||||
|
||||
|
||||
def _extract_segments(meta, base):
|
||||
"""Read (offset_bytes, block_stride_bytes, zero_span_bytes) per segment.
|
||||
|
||||
Accepts both the pre-fix five-field ``_meta`` (no block stride; the zero
|
||||
width is the FULL packed row -- the defect) and the fixed six-field ``_meta``.
|
||||
All values come from the real constructor output; nothing is recomputed. Returns
|
||||
``(segs, legacy_why)`` where ``legacy_why`` describes the pre-fix defect
|
||||
when detected.
|
||||
"""
|
||||
legacy_why = None
|
||||
if len(meta) == 6:
|
||||
seg_addrs, seg_block_strides, seg_page_sizes, _max_chunks, _blk, n_segs = meta
|
||||
else:
|
||||
seg_addrs, seg_page_sizes, _max_chunks, _blk, n_segs = meta
|
||||
seg_block_strides = seg_page_sizes
|
||||
legacy_why = (
|
||||
f"legacy {len(meta)}-field _meta (no block stride): zero width is "
|
||||
f"the FULL packed row ({int(seg_page_sizes[0]) * 4} B) -- the defect"
|
||||
)
|
||||
segs = [
|
||||
(int(a) - base, int(bs) * 4, int(ps) * 4)
|
||||
for a, bs, ps in zip(seg_addrs, seg_block_strides, seg_page_sizes)
|
||||
]
|
||||
return segs, legacy_why, n_segs
|
||||
|
||||
|
||||
def test_dsv4_packed_zeroer_geometry():
|
||||
"""Real DSV4 constructor metadata separates block stride from zero span."""
|
||||
layer_names = [f"model.layers.{i}.self_attn" for i in range(4)]
|
||||
|
||||
specs = [
|
||||
MLAAttentionSpec(
|
||||
block_size=BLOCK_SIZE_CFG,
|
||||
num_kv_heads=1,
|
||||
head_size=512,
|
||||
dtype=torch.uint8,
|
||||
compress_ratio=COMPRESS,
|
||||
cache_dtype_str="fp8_ds_mla",
|
||||
alignment=576,
|
||||
model_version="deepseek_v4",
|
||||
state_content_bytes=584, # >576 to allocate room for scales at back of page
|
||||
)
|
||||
for _ in layer_names
|
||||
]
|
||||
uniform = UniformTypeKVCacheSpecs.from_specs(
|
||||
{ln: sp for ln, sp in zip(layer_names, specs)}
|
||||
def test_packed_dsv4_zeroer_zeroes_only_each_layers_page():
|
||||
spec = MLAAttentionSpec(
|
||||
block_size=256,
|
||||
num_kv_heads=1,
|
||||
head_size=512,
|
||||
dtype=torch.uint8,
|
||||
cache_dtype_str="fp8_ds_mla",
|
||||
model_version="deepseek_v4",
|
||||
tokens_per_state=4,
|
||||
alignment=ALIGNMENT,
|
||||
# DeepseekV4 fp8_ds_mla: 584B per token, published by the layer.
|
||||
state_content_bytes=584,
|
||||
)
|
||||
assert uniform is not None
|
||||
# DSV4: 448B NoPE + 128B RoPE + 8B fp8 scale = 584B per stored state.
|
||||
assert spec.state_content_size_bytes == 584
|
||||
assert spec.num_states == 64
|
||||
unpadded_page = spec.unpadded_page_size_bytes
|
||||
padded_page = spec.page_size_bytes
|
||||
assert unpadded_page == 64 * 584
|
||||
assert padded_page > unpadded_page, "alignment must pad the page"
|
||||
|
||||
# --- Real packed layout planner -------------------------------------------
|
||||
block_stride, layers_by_offset = _get_packed_kv_cache_layout(
|
||||
[
|
||||
AttentionGroup(
|
||||
backend=DeepseekV4FlashMLABackend,
|
||||
layer_names=layer_names,
|
||||
kv_cache_spec=uniform,
|
||||
kv_cache_group_id=0,
|
||||
)
|
||||
]
|
||||
)
|
||||
page_bytes = specs[0].page_size_bytes # DSV4 fp8_ds_mla page after 576B alignment
|
||||
real_page_bytes = specs[0].real_page_size_bytes # unpadded meaningful page
|
||||
alignment_gap = page_bytes - real_page_bytes # trailing planner padding
|
||||
total_size = block_stride * NUM_BLOCKS
|
||||
offsets = sorted(layers_by_offset)
|
||||
assert block_stride == 4 * page_bytes, (block_stride, 4 * page_bytes)
|
||||
assert offsets == [i * page_bytes for i in range(4)]
|
||||
raw = torch.zeros(NUM_BLOCKS * NUM_LAYERS * padded_page, dtype=torch.int8)
|
||||
views = dense_kv_cache_views(raw, spec, NUM_BLOCKS, NUM_LAYERS, KVCacheLayout.BLHNC)
|
||||
base = raw.data_ptr()
|
||||
block_row = NUM_LAYERS * padded_page
|
||||
for i, view in enumerate(views):
|
||||
assert view.data_ptr() - base == i * padded_page
|
||||
assert view.stride(0) * view.element_size() == block_row
|
||||
|
||||
# --- Real per-layer views (packed branch) --------------------------------
|
||||
backing = torch.zeros(total_size, dtype=torch.uint8)
|
||||
base = backing.data_ptr()
|
||||
views = {}
|
||||
for ln, off in zip(layer_names, offsets):
|
||||
packing = (off, block_stride)
|
||||
kv_shape = DeepseekV4FlashMLABackend.get_kv_cache_shape(
|
||||
NUM_BLOCKS, STORAGE_BS, 1, 512, cache_dtype_str="fp8_ds_mla"
|
||||
)
|
||||
stride_order = tuple(range(len(kv_shape)))
|
||||
view = _reshape_attention_kv_cache(
|
||||
backing, specs[0], kv_shape, stride_order, NUM_BLOCKS, packing
|
||||
)
|
||||
views[ln] = view
|
||||
assert view.data_ptr() - base == off, "data_ptr must be base+offset"
|
||||
assert view.stride(0) * view.element_size() == block_stride, (
|
||||
"row stride == block_stride (logical block stride)"
|
||||
)
|
||||
|
||||
# --- Real KVBlockZeroer.__init__ (production arithmetic, CPU) ----------
|
||||
sctx = {ln: SimpleNamespace(kv_cache=v) for ln, v in views.items()}
|
||||
zeroer = KVBlockZeroer(
|
||||
torch.device("cpu"),
|
||||
attn_groups_iter=[
|
||||
AttentionGroup(
|
||||
backend=DeepseekV4FlashMLABackend,
|
||||
layer_names=layer_names,
|
||||
kv_cache_spec=specs[0],
|
||||
kv_cache_group_id=0,
|
||||
)
|
||||
],
|
||||
kernel_block_sizes=[KERNEL_BS],
|
||||
cache_dtype="fp8_ds_mla",
|
||||
static_forward_context=sctx,
|
||||
attn_groups_iter=iter(
|
||||
[
|
||||
AttentionGroup(
|
||||
backend=None,
|
||||
layer_names=[f"layer.{i}" for i in range(NUM_LAYERS)],
|
||||
kv_cache_spec=spec,
|
||||
kv_cache_group_id=0,
|
||||
)
|
||||
]
|
||||
),
|
||||
kernel_block_sizes=[spec.block_size],
|
||||
static_forward_context={
|
||||
f"layer.{i}": SimpleNamespace(kv_cache=views[i]) for i in range(NUM_LAYERS)
|
||||
},
|
||||
)
|
||||
segs, legacy_why, n_segs = _extract_segments(zeroer._meta, base)
|
||||
seg_addrs, seg_block_strides, seg_page_sizes, _, _, n_segs = zeroer._meta
|
||||
|
||||
# --- Fixed semantics (proven by metadata values, not by schema shape) ----
|
||||
assert n_segs == 4, "all 4 packed layers must register as segments"
|
||||
assert [off for off, _, _ in segs] == offsets, "one segment per layer offset"
|
||||
assert all(bs == block_stride for _, bs, _ in segs), (
|
||||
"block stride must be the full packed row (4 aligned pages)"
|
||||
assert n_segs == NUM_LAYERS
|
||||
# Segments step by the full packed row per block...
|
||||
assert (seg_block_strides * 4 == block_row).all()
|
||||
# ...but zero only the layer's meaningful page: no alignment padding, no
|
||||
# adjacent layer's page.
|
||||
assert (seg_page_sizes * 4 == unpadded_page).all()
|
||||
assert sorted(a - base for a in seg_addrs.tolist()) == [
|
||||
i * padded_page for i in range(NUM_LAYERS)
|
||||
]
|
||||
# Block 99 of the highest-offset layer stays within the packed backing.
|
||||
last_end = (
|
||||
max(seg_addrs.tolist())
|
||||
+ (NUM_BLOCKS - 1) * int(seg_block_strides[0]) * 4
|
||||
+ int(seg_page_sizes[0]) * 4
|
||||
)
|
||||
assert all(zs == real_page_bytes for _, _, zs in segs), (
|
||||
f"zero span must be the meaningful page ({real_page_bytes} B), got "
|
||||
f"{[zs for _, _, zs in segs]} B" + (f"; {legacy_why}" if legacy_why else "")
|
||||
assert last_end <= base + raw.numel()
|
||||
|
||||
|
||||
def test_overlaid_zeroer_dedups_segments_with_max_span():
|
||||
"""Two groups overlay one allocation; the zeroer must emit one segment per distinct
|
||||
byte offset, spanning the widest overlaid page, so a newly allocated block is fully
|
||||
zeroed no matter which group owns it."""
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from vllm.v1.core.kv_cache_utils import get_kv_cache_config_from_groups
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
KVCacheGroupSpec,
|
||||
KVCacheLayout,
|
||||
UniformTypeKVCacheSpecs,
|
||||
)
|
||||
|
||||
# --- Block 99 (the real final block) bounds, via constructor metadata ---
|
||||
# Production write for block b of a segment is
|
||||
# [off + b*block_stride_bytes, off + b*block_stride_bytes + zero_span_bytes).
|
||||
worst = 0
|
||||
for off, bs, zs in segs:
|
||||
start = off + LAST_BLOCK * bs
|
||||
end = start + zs
|
||||
oob = end - total_size
|
||||
worst = max(worst, oob)
|
||||
assert end <= total_size, (
|
||||
f"segment@off={off} block {LAST_BLOCK} writes [{start}, {end}) "
|
||||
f"-> {oob} bytes past backing ({total_size})"
|
||||
def make_spec(head_size):
|
||||
return MLAAttentionSpec(
|
||||
block_size=64, num_kv_heads=1, head_size=head_size, dtype=torch.uint8
|
||||
)
|
||||
assert worst == 0, f"highest-offset final block goes {worst} bytes OOB"
|
||||
|
||||
# Highest-offset segment's block 99 is the last written region and ends exactly at
|
||||
# total_size - alignment_gap: the trailing planner padding stays untouched.
|
||||
off_last, bs_last, zs_last = segs[-1]
|
||||
assert off_last + LAST_BLOCK * bs_last + zs_last == total_size - alignment_gap, (
|
||||
"highest-offset final block must end at total_size - alignment_gap "
|
||||
f"({alignment_gap}-byte gap intact)"
|
||||
g1_specs = {"g1.big": make_spec(512), "g1.small": make_spec(128)}
|
||||
g2_specs = {"g2.huge": make_spec(1024)}
|
||||
groups = [
|
||||
KVCacheGroupSpec(
|
||||
list(g1_specs),
|
||||
UniformTypeKVCacheSpecs(block_size=64, kv_cache_specs=g1_specs),
|
||||
),
|
||||
KVCacheGroupSpec(
|
||||
list(g2_specs),
|
||||
UniformTypeKVCacheSpecs(block_size=64, kv_cache_specs=g2_specs),
|
||||
),
|
||||
]
|
||||
from vllm.config import CacheConfig
|
||||
|
||||
vllm_config = MagicMock()
|
||||
vllm_config.cache_config = CacheConfig()
|
||||
vllm_config.cache_config.num_gpu_blocks_override = None
|
||||
vllm_config.cache_config.kv_cache_layout = "BLHNC"
|
||||
config = get_kv_cache_config_from_groups(vllm_config, groups, 8 * 1024 * 1024)
|
||||
views = allocate_kv_cache(config, torch.device("cpu"), KVCacheLayout.BLHNC, None)
|
||||
buf_ptr = views["g1.big"].data_ptr()
|
||||
|
||||
attn_groups = [
|
||||
AttentionGroup(
|
||||
backend=None,
|
||||
layer_names=list(specs),
|
||||
kv_cache_spec=next(iter(specs.values())),
|
||||
kv_cache_group_id=gid,
|
||||
)
|
||||
for gid, specs in enumerate((g1_specs, g2_specs))
|
||||
]
|
||||
zeroer = KVBlockZeroer(
|
||||
torch.device("cpu"),
|
||||
attn_groups_iter=iter(attn_groups),
|
||||
kernel_block_sizes=[64, 64],
|
||||
static_forward_context={
|
||||
name: SimpleNamespace(kv_cache=views[name]) for name in views
|
||||
},
|
||||
)
|
||||
seg_addrs, seg_block_strides, seg_page_sizes, _, _, n_segs = zeroer._meta
|
||||
|
||||
# Zero span < block stride proves adjacent packed layers are never wiped.
|
||||
assert max(zs for _, _, zs in segs) < block_stride
|
||||
# g1.big and g2.huge overlay at offset 0 -> one segment with g2's wider
|
||||
# span; g1.small keeps its own segment.
|
||||
assert n_segs == 2
|
||||
pages = {n: s.page_size_bytes for n, s in (g1_specs | g2_specs).items()}
|
||||
by_offset = {
|
||||
a - buf_ptr: p * 4 for a, p in zip(seg_addrs.tolist(), seg_page_sizes.tolist())
|
||||
}
|
||||
assert by_offset[0] == max(pages["g1.big"], pages["g2.huge"])
|
||||
assert by_offset[pages["g1.big"]] == pages["g1.small"]
|
||||
packed_block_stride = max(sum(pages[n] for n in g) for g in (g1_specs, g2_specs))
|
||||
assert (seg_block_strides * 4 == packed_block_stride).all()
|
||||
|
||||
@@ -92,7 +92,12 @@ def initialize_kv_cache(runner: GPUModelRunner):
|
||||
kv_cache_config = KVCacheConfig(
|
||||
num_blocks=NUM_BLOCKS,
|
||||
kv_cache_tensors=[
|
||||
KVCacheTensor(size=tensor_size, shared_by=["layer.0"]),
|
||||
KVCacheTensor(
|
||||
size=tensor_size,
|
||||
layers=["layer.0"],
|
||||
layer_stride=tensor_size,
|
||||
block_stride=attn_spec.page_size_bytes,
|
||||
),
|
||||
],
|
||||
kv_cache_groups=[
|
||||
KVCacheGroupSpec(layer_names=["layer.0"], kv_cache_spec=attn_spec)
|
||||
@@ -131,6 +136,7 @@ def get_vllm_config():
|
||||
gpu_memory_utilization=0.9,
|
||||
cache_dtype="auto",
|
||||
)
|
||||
cache_config.kv_cache_layout = "LBNHC"
|
||||
parallel_config = ParallelConfig()
|
||||
vllm_config = VllmConfig(
|
||||
model_config=model_config,
|
||||
@@ -839,56 +845,6 @@ def test_update_states_pp_async_multi_request_keeps_rank_state_consistent(
|
||||
)
|
||||
|
||||
|
||||
def test_kv_cache_stride_order(monkeypatch, model_runner):
|
||||
# This test checks if GPUModelRunner initializes correctly when an attention
|
||||
# backend enforces a non-default KV cache stride order.
|
||||
n_heads = model_runner.model_config.get_num_kv_heads(model_runner.parallel_config)
|
||||
head_size = model_runner.model_config.get_head_size()
|
||||
|
||||
# Get the expected shape from the backend's get_kv_cache_shape method
|
||||
# to ensure compatibility with different backends (triton vs flexattention)
|
||||
attn_backend = None
|
||||
for attn_group in model_runner._attn_group_iterator():
|
||||
attn_backend = attn_group.backend
|
||||
break
|
||||
|
||||
assert attn_backend is not None, "No attention backend found"
|
||||
expected_kv_cache_shape = list(
|
||||
attn_backend.get_kv_cache_shape(NUM_BLOCKS, BLOCK_SIZE, n_heads, head_size)
|
||||
)
|
||||
|
||||
# TODO mla test
|
||||
default_stride = tuple(range(len(expected_kv_cache_shape)))
|
||||
non_default_stride = (*default_stride[1:], default_stride[0])
|
||||
# Permutation that gets you back to expected kv shape
|
||||
for test_stride in (non_default_stride, default_stride):
|
||||
|
||||
def rnd_stride_order(
|
||||
include_num_layers_dimension: bool = False, test_stride=test_stride
|
||||
):
|
||||
assert not include_num_layers_dimension
|
||||
return test_stride
|
||||
|
||||
# Patch the attention backend class and re-trigger the KV cache creation
|
||||
for attn_group in model_runner._attn_group_iterator():
|
||||
attn_backend = attn_group.backend
|
||||
monkeypatch.setattr(
|
||||
attn_backend, "get_kv_cache_stride_order", rnd_stride_order
|
||||
)
|
||||
|
||||
model_runner.attn_groups = []
|
||||
model_runner.kv_caches = []
|
||||
model_runner.initialize_kv_cache(model_runner.kv_cache_config)
|
||||
|
||||
# Shape is unchanged, but layout may differ
|
||||
kv_cache_shape = model_runner.kv_caches[0].shape
|
||||
assert list(kv_cache_shape) == expected_kv_cache_shape
|
||||
if default_stride == test_stride:
|
||||
assert all(kv.is_contiguous() for kv in model_runner.kv_caches)
|
||||
else:
|
||||
assert all(not kv.is_contiguous() for kv in model_runner.kv_caches)
|
||||
|
||||
|
||||
def test_update_config(model_runner):
|
||||
# Simple update
|
||||
model_runner.update_config({"load_config": {"load_format": "dummy"}})
|
||||
@@ -1124,21 +1080,22 @@ def test_init_kv_cache_without_kv_sharing(default_vllm_config):
|
||||
vllm_config, [kv_cache_spec], [available_memory]
|
||||
)[0]
|
||||
assert kv_cache_config.num_blocks == num_expected_blocks
|
||||
assert len(kv_cache_config.kv_cache_tensors) == 2
|
||||
assert kv_cache_config.kv_cache_tensors[0].size == available_memory // 2
|
||||
assert kv_cache_config.kv_cache_tensors[1].size == available_memory // 2
|
||||
assert len(kv_cache_config.kv_cache_tensors) == 1
|
||||
assert kv_cache_config.kv_cache_tensors[0].size == available_memory
|
||||
|
||||
max_context_len = estimate_max_model_len(vllm_config, kv_cache_spec, 5 * GiB_bytes)
|
||||
# max context len with KV sharing should be 2x as large as without
|
||||
assert max_context_len == 1310720
|
||||
|
||||
# important: override tensor size to prevent large mem alloc during test
|
||||
# this will only allocate 2 block worth of memory (2 * 32kb)
|
||||
# this will only allocate 1 block worth of memory per layer (2 layers * 32kb)
|
||||
kv_cache_config.num_blocks = 1
|
||||
for kv_cache_tensor in kv_cache_config.kv_cache_tensors:
|
||||
kv_cache_tensor.size = kv_cache_spec[
|
||||
kv_cache_tensor.shared_by[0]
|
||||
].page_size_bytes
|
||||
page_size = kv_cache_spec[kv_cache_tensor.layers[0]].page_size_bytes
|
||||
kv_cache_tensor.size = page_size * len(kv_cache_tensor.layers)
|
||||
# One block per layer: a layer's region is exactly one page.
|
||||
kv_cache_tensor.layer_stride = page_size
|
||||
kv_cache_tensor.block_stride = page_size
|
||||
|
||||
runner.initialize_kv_cache(kv_cache_config)
|
||||
|
||||
@@ -1207,7 +1164,9 @@ def test_init_kv_cache_with_kv_sharing_valid(default_vllm_config):
|
||||
# important: override tensor size to prevent large mem alloc during test
|
||||
# this will only allocate 1 block worth of memory (32kb)
|
||||
kv_cache_config.num_blocks = 1
|
||||
kv_cache_config.kv_cache_tensors[0].size = kv_cache_spec[layer_0].page_size_bytes
|
||||
page_size = kv_cache_spec[layer_0].page_size_bytes
|
||||
kv_cache_config.kv_cache_tensors[0].size = page_size
|
||||
kv_cache_config.kv_cache_tensors[0].layer_stride = page_size
|
||||
|
||||
runner.initialize_kv_cache(kv_cache_config)
|
||||
kv_cache_config_after_init = runner.kv_cache_config
|
||||
@@ -1232,7 +1191,7 @@ def test_hybrid_attention_mamba_tensor_shapes():
|
||||
"""
|
||||
The GPU model runner creates different views into the
|
||||
KVCacheTensors for the attention and mamba layers
|
||||
(via _reshape_kv_cache_tensors function). This test verifies
|
||||
(via _allocate_kv_caches). This test verifies
|
||||
that the views are compatible: writing a mamba block
|
||||
will not corrupt an attention block and vice versa
|
||||
"""
|
||||
@@ -1270,6 +1229,7 @@ def test_hybrid_attention_mamba_tensor_shapes():
|
||||
gpu_memory_utilization=0.9,
|
||||
cache_dtype="auto",
|
||||
)
|
||||
cache_config.kv_cache_layout = "LBNHC"
|
||||
parallel_config = ParallelConfig()
|
||||
attention_config = AttentionConfig(backend=AttentionBackendEnum.FLASHINFER)
|
||||
vllm_config = VllmConfig(
|
||||
@@ -1561,7 +1521,12 @@ def test_hybrid_cache_integration(default_vllm_config, dist_init):
|
||||
kv_cache_config = KVCacheConfig(
|
||||
num_blocks=NUM_BLOCKS,
|
||||
kv_cache_tensors=[
|
||||
KVCacheTensor(size=tensor_size, shared_by=["layer.0"]),
|
||||
KVCacheTensor(
|
||||
size=tensor_size,
|
||||
layers=["layer.0"],
|
||||
layer_stride=tensor_size,
|
||||
block_stride=attn_spec.page_size_bytes,
|
||||
),
|
||||
],
|
||||
kv_cache_groups=[
|
||||
KVCacheGroupSpec(layer_names=["layer.0"], kv_cache_spec=attn_spec)
|
||||
@@ -1726,6 +1691,7 @@ def test_mamba_cache_raises_when_max_num_seqs_exceeds_blocks():
|
||||
gpu_memory_utilization=0.9,
|
||||
cache_dtype="auto",
|
||||
)
|
||||
cache_config.kv_cache_layout = "LBNHC"
|
||||
parallel_config = ParallelConfig()
|
||||
attention_config = AttentionConfig(backend=AttentionBackendEnum.FLASHINFER)
|
||||
vllm_config = VllmConfig(
|
||||
|
||||
@@ -7,8 +7,11 @@ from types import SimpleNamespace
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from tests.v1.attention.utils import dense_kv_cache_views
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
ChunkedLocalAttentionSpec,
|
||||
FullAttentionSpec,
|
||||
KVCacheLayout,
|
||||
SlidingWindowSpec,
|
||||
)
|
||||
from vllm.v1.worker import utils as worker_utils
|
||||
@@ -19,12 +22,6 @@ from vllm.v1.worker.utils import (
|
||||
)
|
||||
|
||||
|
||||
class _BlockFirstBackend:
|
||||
@staticmethod
|
||||
def get_kv_cache_block_dim(*args, **kwargs):
|
||||
return 0
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required")
|
||||
@pytest.mark.parametrize(
|
||||
"spec",
|
||||
@@ -52,11 +49,8 @@ def test_attention_blocks_are_zeroed(spec):
|
||||
layer_name = "draft.self_attn"
|
||||
zeroer = KVBlockZeroer(
|
||||
device,
|
||||
attn_groups_iter=[
|
||||
AttentionGroup(_BlockFirstBackend, [layer_name], spec, 0) # type: ignore[arg-type]
|
||||
],
|
||||
attn_groups_iter=[AttentionGroup(None, [layer_name], spec, 0)],
|
||||
kernel_block_sizes=[2],
|
||||
cache_dtype="fp8",
|
||||
static_forward_context={
|
||||
layer_name: SimpleNamespace(kv_cache=storage),
|
||||
},
|
||||
@@ -205,16 +199,10 @@ def test_large_dsv4_launch_geometry(monkeypatch):
|
||||
zeroer = KVBlockZeroer(
|
||||
device,
|
||||
attn_groups_iter=[
|
||||
AttentionGroup(
|
||||
_BlockFirstBackend, # type: ignore[arg-type]
|
||||
[name],
|
||||
spec,
|
||||
group_id,
|
||||
)
|
||||
AttentionGroup(None, [name], spec, group_id)
|
||||
for group_id, name in enumerate(layer_names)
|
||||
],
|
||||
kernel_block_sizes=[1] * n_segs,
|
||||
cache_dtype="auto",
|
||||
static_forward_context={
|
||||
name: SimpleNamespace(kv_cache=storage)
|
||||
for name, storage in storages.items()
|
||||
@@ -311,3 +299,46 @@ def test_warmup_respects_available_block_count():
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
assert torch.all(storage == 1)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required")
|
||||
@pytest.mark.parametrize("layout", list(KVCacheLayout))
|
||||
def test_zeroes_exactly_one_block_per_layer(layout: KVCacheLayout):
|
||||
"""The zeroer must zero every byte of the target block in every layer and nothing
|
||||
outside it — per head-group region under LHBNC, and never past the target block's
|
||||
tile under block-major layouts (no out-of-bounds writes, no clobbering)."""
|
||||
device = torch.device("cuda")
|
||||
num_blocks, num_layers = 4, 2
|
||||
spec = FullAttentionSpec(
|
||||
block_size=4, num_kv_heads=2, head_size=8, dtype=torch.float32
|
||||
)
|
||||
raw = torch.empty(
|
||||
num_blocks * num_layers * spec.page_size_bytes,
|
||||
dtype=torch.int8,
|
||||
device=device,
|
||||
).fill_(1)
|
||||
views = dense_kv_cache_views(raw, spec, num_blocks, num_layers, layout)
|
||||
groups = [
|
||||
AttentionGroup(
|
||||
backend=None,
|
||||
layer_names=[f"layer.{i}" for i in range(num_layers)],
|
||||
kv_cache_spec=spec,
|
||||
kv_cache_group_id=0,
|
||||
)
|
||||
]
|
||||
ctx = {f"layer.{i}": SimpleNamespace(kv_cache=views[i]) for i in range(num_layers)}
|
||||
zeroer = KVBlockZeroer(
|
||||
device,
|
||||
attn_groups_iter=iter(groups),
|
||||
kernel_block_sizes=[spec.block_size],
|
||||
static_forward_context=ctx,
|
||||
)
|
||||
zeroer.zero_block_ids([2])
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
for view in views:
|
||||
assert (view[2] == 0).all(), layout
|
||||
for b in (0, 1, 3):
|
||||
assert (view[b].view(torch.int8) == 1).all(), layout
|
||||
zero_bytes = int((raw == 0).sum().item())
|
||||
assert zero_bytes == num_layers * spec.page_size_bytes, layout
|
||||
|
||||
Reference in New Issue
Block a user