[Bugfix][ROCm] Give KV-first attention blocks their own page in hybrid models (#51837)
Signed-off-by: Stefan Koncarevic <stefan.koncarevic@amd.com>
This commit is contained in:
@@ -3,7 +3,13 @@
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.v1.kv_cache_interface import FullAttentionSpec, KVQuantMode
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
FullAttentionSpec,
|
||||
KVCacheConfig,
|
||||
KVCacheTensor,
|
||||
KVQuantMode,
|
||||
MambaSpec,
|
||||
)
|
||||
from vllm.v1.worker.gpu.attn_utils import _reshape_kv_cache
|
||||
from vllm.v1.worker.utils import AttentionGroup
|
||||
|
||||
@@ -240,3 +246,99 @@ def test_reshape_padded_quantized_kv_cache_preserves_scale_stride():
|
||||
assert kv_cache.stride(0) == spec.page_size_bytes
|
||||
assert kv_cache.stride(1) == 16 * 1 * 8
|
||||
assert kv_cache[1, 1].storage_offset() == spec.page_size_bytes + 16 * 1 * 8
|
||||
|
||||
|
||||
class FakeKVFirstBackend:
|
||||
"""ROCm-style backend that puts K and V ahead of the block dim."""
|
||||
|
||||
@staticmethod
|
||||
def get_kv_cache_shape(
|
||||
num_blocks: int,
|
||||
block_size: int,
|
||||
num_kv_heads: int,
|
||||
head_size: int,
|
||||
cache_dtype_str: str = "auto",
|
||||
) -> tuple[int, ...]:
|
||||
return (2, num_blocks, block_size, num_kv_heads, head_size)
|
||||
|
||||
@staticmethod
|
||||
def get_kv_cache_block_dim(
|
||||
block_size: int,
|
||||
num_kv_heads: int,
|
||||
head_size: int,
|
||||
cache_dtype_str: str = "auto",
|
||||
) -> int:
|
||||
return 1
|
||||
|
||||
|
||||
def _kv_first_setup(shared_by: list[str]):
|
||||
num_blocks = 3
|
||||
attn_spec = FullAttentionSpec(
|
||||
block_size=16,
|
||||
num_kv_heads=1,
|
||||
head_size=2,
|
||||
dtype=torch.float32,
|
||||
)
|
||||
mamba_spec = MambaSpec(
|
||||
block_size=16,
|
||||
shapes=((64,),),
|
||||
dtypes=(torch.float32,),
|
||||
)
|
||||
assert attn_spec.page_size_bytes == mamba_spec.page_size_bytes == 256
|
||||
|
||||
raw_tensor = torch.zeros(attn_spec.page_size_bytes * num_blocks, dtype=torch.int8)
|
||||
raw_tensors = {name: raw_tensor for name in shared_by}
|
||||
attn_groups = [
|
||||
AttentionGroup(
|
||||
backend=FakeKVFirstBackend,
|
||||
layer_names=["attn"],
|
||||
kv_cache_spec=attn_spec,
|
||||
kv_cache_group_id=0,
|
||||
)
|
||||
]
|
||||
if "mamba" in shared_by:
|
||||
attn_groups.append(
|
||||
AttentionGroup(
|
||||
backend=FakeKVFirstBackend,
|
||||
layer_names=["mamba"],
|
||||
kv_cache_spec=mamba_spec,
|
||||
kv_cache_group_id=1,
|
||||
)
|
||||
)
|
||||
kv_cache_config = KVCacheConfig(
|
||||
num_blocks=num_blocks,
|
||||
kv_cache_tensors=[
|
||||
KVCacheTensor(size=raw_tensor.numel(), shared_by=list(shared_by))
|
||||
],
|
||||
kv_cache_groups=[],
|
||||
)
|
||||
|
||||
kv_caches = _reshape_kv_cache(
|
||||
attn_groups,
|
||||
raw_tensors,
|
||||
"auto",
|
||||
[attn_spec.block_size] * len(attn_groups),
|
||||
{},
|
||||
kv_cache_config,
|
||||
)
|
||||
return num_blocks, kv_caches["attn"]
|
||||
|
||||
|
||||
def test_reshape_kv_first_kv_cache_pages_blocks_when_shared_with_mamba():
|
||||
num_blocks, kv_cache = _kv_first_setup(["attn", "mamba"])
|
||||
|
||||
assert kv_cache.shape == (2, num_blocks, 16, 1, 2)
|
||||
# Block b has to own page b, so its K and V sit side by side within it.
|
||||
page = 16 * 1 * 2 * 2
|
||||
for block in range(num_blocks):
|
||||
assert kv_cache[0, block].storage_offset() == block * page
|
||||
assert kv_cache[1, block].storage_offset() == block * page + page // 2
|
||||
|
||||
|
||||
def test_reshape_kv_first_kv_cache_keeps_layout_without_mamba():
|
||||
num_blocks, kv_cache = _kv_first_setup(["attn"])
|
||||
|
||||
assert kv_cache.shape == (2, num_blocks, 16, 1, 2)
|
||||
# Nothing else indexes this allocation by page, so K and V stay split into
|
||||
# one contiguous half each.
|
||||
assert kv_cache[1, 0].storage_offset() == num_blocks * 16 * 1 * 2
|
||||
|
||||
Reference in New Issue
Block a user