[6/N][KV-Cache Layout Refactor] Standardize KV cache layout (#51718)
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
This commit is contained in:
@@ -7,7 +7,11 @@ import torch._dynamo
|
||||
|
||||
from tests.compile.backend import LazyInitPass, TestBackend
|
||||
from tests.utils import TestFP8Layer, flat_product
|
||||
from tests.v1.attention.utils import BatchSpec, create_common_attn_metadata
|
||||
from tests.v1.attention.utils import (
|
||||
BatchSpec,
|
||||
create_common_attn_metadata,
|
||||
dense_kv_cache_views,
|
||||
)
|
||||
from vllm._custom_ops import cutlass_scaled_fp4_mm, scaled_fp4_quant
|
||||
from vllm.compilation.passes.fusion.attn_quant_fusion import (
|
||||
ATTN_OP,
|
||||
@@ -39,7 +43,14 @@ from vllm.platforms import current_platform
|
||||
from vllm.utils.flashinfer import has_flashinfer
|
||||
from vllm.v1.attention.backend import AttentionMetadata
|
||||
from vllm.v1.attention.backends.registry import AttentionBackendEnum
|
||||
from vllm.v1.kv_cache_interface import AttentionSpec, get_kv_quant_mode
|
||||
from vllm.v1.attention.backends.utils import (
|
||||
get_supported_kv_cache_layouts,
|
||||
resolve_kv_cache_layout,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
AttentionSpec,
|
||||
get_kv_quant_mode,
|
||||
)
|
||||
|
||||
DEVICE_TYPE = current_platform.device_type
|
||||
FP8_DTYPE = current_platform.fp8_dtype()
|
||||
@@ -108,32 +119,29 @@ class AttentionQuantPatternModel(torch.nn.Module):
|
||||
max_blocks = (max(batch_spec.seq_lens) + self.block_size - 1) // self.block_size
|
||||
num_blocks = batch_size * max_blocks
|
||||
|
||||
# Fetch the attention backend and kv cache shape and stride order
|
||||
attn_backend = self.attn.attn_backend
|
||||
kv_cache_shape = attn_backend.get_kv_cache_shape(
|
||||
num_blocks,
|
||||
self.block_size,
|
||||
self.num_kv_heads,
|
||||
self.head_size,
|
||||
cache_dtype_str=self.attn.kv_cache_dtype,
|
||||
)
|
||||
try:
|
||||
kv_cache_stride_order = attn_backend.get_kv_cache_stride_order()
|
||||
except (AttributeError, NotImplementedError):
|
||||
kv_cache_stride_order = tuple(range(len(kv_cache_shape)))
|
||||
|
||||
kv_cache_shape = tuple(kv_cache_shape[i] for i in kv_cache_stride_order)
|
||||
inv_order = [
|
||||
kv_cache_stride_order.index(i) for i in range(len(kv_cache_stride_order))
|
||||
]
|
||||
|
||||
# Create dummy KV cache
|
||||
raw_tensor = torch.zeros(
|
||||
kv_cache_shape,
|
||||
spec = AttentionSpec(
|
||||
block_size=self.block_size,
|
||||
num_kv_heads=self.num_kv_heads,
|
||||
head_size=self.head_size,
|
||||
dtype=self.attn.kv_cache_torch_dtype,
|
||||
kv_quant_mode=get_kv_quant_mode(self.attn.kv_cache_dtype),
|
||||
)
|
||||
supported = get_supported_kv_cache_layouts([self.attn.attn_backend])
|
||||
layout = resolve_kv_cache_layout(
|
||||
self.vllm_config, [[m.name for m in supported]]
|
||||
)
|
||||
raw_tensor = torch.zeros(
|
||||
num_blocks * spec.page_size_bytes,
|
||||
dtype=torch.int8,
|
||||
device=self.device,
|
||||
)
|
||||
kv_cache = raw_tensor.permute(*inv_order)
|
||||
kv_cache = dense_kv_cache_views(
|
||||
raw_tensor,
|
||||
spec,
|
||||
num_blocks,
|
||||
num_layers=1,
|
||||
layout=layout,
|
||||
)[0]
|
||||
|
||||
self.attn.kv_cache = kv_cache
|
||||
|
||||
|
||||
@@ -150,27 +150,14 @@ class MLAAttentionQuantPatternModel(torch.nn.Module):
|
||||
max_blocks = (max(batch_spec.seq_lens) + self.block_size - 1) // self.block_size
|
||||
num_blocks = batch_size * max_blocks
|
||||
|
||||
# MLA KV cache is 3D: (num_blocks, block_size, head_size)
|
||||
attn_backend = self.mla_attn.attn_backend
|
||||
kv_cache_shape = attn_backend.get_kv_cache_shape(
|
||||
num_blocks, self.block_size, 1, self.head_size
|
||||
# MLA KV cache is 4D: (num_blocks, num_heads=1, block_size, head_size)
|
||||
kv_cache = torch.zeros(
|
||||
(num_blocks, 1, self.block_size, self.head_size),
|
||||
dtype=self.kv_cache_dtype,
|
||||
device=self.device,
|
||||
)
|
||||
try:
|
||||
kv_cache_stride_order = attn_backend.get_kv_cache_stride_order()
|
||||
except (AttributeError, NotImplementedError):
|
||||
kv_cache_stride_order = tuple(range(len(kv_cache_shape)))
|
||||
|
||||
ordered_shape = tuple(kv_cache_shape[i] for i in kv_cache_stride_order)
|
||||
inv_order = [
|
||||
kv_cache_stride_order.index(i) for i in range(len(kv_cache_stride_order))
|
||||
]
|
||||
|
||||
raw_tensor = torch.zeros(
|
||||
ordered_shape, dtype=self.kv_cache_dtype, device=self.device
|
||||
)
|
||||
kv_cache = raw_tensor.permute(*inv_order)
|
||||
|
||||
self.mla_attn.kv_cache = kv_cache
|
||||
self.mla_attn.bind_kv_cache(kv_cache)
|
||||
|
||||
self.attn_metadata = self.builder.build(
|
||||
common_prefix_len=0, common_attn_metadata=common_attn_metadata
|
||||
|
||||
@@ -165,29 +165,14 @@ class MLARoPEKVCacheCatTestModel(torch.nn.Module):
|
||||
max_blocks = (max(batch_spec.seq_lens) + self.block_size - 1) // self.block_size
|
||||
num_blocks = batch_size * max_blocks
|
||||
|
||||
# Fetch the attention backend and kv cache shape and stride order
|
||||
kv_cache_shape = self.attn_backend.get_kv_cache_shape(
|
||||
num_blocks, self.block_size, self.num_kv_heads, self.head_size
|
||||
)
|
||||
try:
|
||||
kv_cache_stride_order = self.attn_backend.get_kv_cache_stride_order()
|
||||
except (AttributeError, NotImplementedError):
|
||||
kv_cache_stride_order = tuple(range(len(kv_cache_shape)))
|
||||
|
||||
kv_cache_shape = tuple(kv_cache_shape[i] for i in kv_cache_stride_order)
|
||||
inv_order = [
|
||||
kv_cache_stride_order.index(i) for i in range(len(kv_cache_stride_order))
|
||||
]
|
||||
|
||||
raw_tensor = torch.zeros(
|
||||
num_blocks * self.block_size * self.num_kv_heads * self.head_size,
|
||||
# MLA KV cache is 4D: (num_blocks, num_heads=1, block_size, head_size)
|
||||
kv_cache = torch.zeros(
|
||||
(num_blocks, 1, self.block_size, self.head_size),
|
||||
dtype=self.kv_cache_dtype,
|
||||
device=self.device,
|
||||
)
|
||||
raw_tensor = raw_tensor.view(kv_cache_shape)
|
||||
kv_cache = raw_tensor.permute(*inv_order)
|
||||
|
||||
self.mla_attn.kv_cache = kv_cache
|
||||
self.mla_attn.bind_kv_cache(kv_cache)
|
||||
|
||||
# Build attn metadata
|
||||
attn_metadata = self.builder.build(
|
||||
|
||||
@@ -10,7 +10,11 @@ from packaging.version import Version
|
||||
|
||||
import vllm.config
|
||||
from tests.compile.backend import TestBackend
|
||||
from tests.v1.attention.utils import BatchSpec, create_common_attn_metadata
|
||||
from tests.v1.attention.utils import (
|
||||
BatchSpec,
|
||||
create_common_attn_metadata,
|
||||
dense_kv_cache_views,
|
||||
)
|
||||
from vllm._aiter_ops import is_aiter_found_and_supported, rocm_aiter_ops
|
||||
from vllm.compilation.passes.fusion.matcher_utils import ROTARY_OP
|
||||
from vllm.compilation.passes.fusion.qk_norm_rope_kvcache_fusion import (
|
||||
@@ -40,7 +44,7 @@ from vllm.v1.attention.backend import (
|
||||
CommonAttentionMetadata,
|
||||
)
|
||||
from vllm.v1.attention.backends.registry import AttentionBackendEnum
|
||||
from vllm.v1.kv_cache_interface import AttentionSpec
|
||||
from vllm.v1.kv_cache_interface import AttentionSpec, KVCacheLayout
|
||||
|
||||
pytestmark = pytest.mark.skip_global_cleanup
|
||||
|
||||
@@ -147,20 +151,25 @@ class QKNormRoPEKVCacheTestModel(torch.nn.Module):
|
||||
self.attn._k_scale = self.attn._k_scale.to(device)
|
||||
self.attn._v_scale = self.attn._v_scale.to(device)
|
||||
|
||||
self.builder = self.attn.attn_backend.get_builder_cls()(
|
||||
kv_cache_spec=AttentionSpec(
|
||||
# Mirror the worker spec-collection loop: the backend publishes the
|
||||
# packing it needs.
|
||||
self.kv_cache_spec = self.attn.attn_backend.customize_spec(
|
||||
AttentionSpec(
|
||||
block_size=self.block_size,
|
||||
num_kv_heads=self.num_kv_heads,
|
||||
head_size=head_size,
|
||||
dtype=self.kv_cache_dtype,
|
||||
),
|
||||
)
|
||||
)
|
||||
self.builder = self.attn.attn_backend.get_builder_cls()(
|
||||
kv_cache_spec=self.kv_cache_spec,
|
||||
layer_names=[self.attn.layer_name],
|
||||
vllm_config=vllm_config,
|
||||
device=device,
|
||||
)
|
||||
|
||||
def build_attn_metadata(
|
||||
self, batch_size: int, kv_stride_order: tuple[int, ...] | None = None
|
||||
self, batch_size: int, layout: KVCacheLayout
|
||||
) -> CommonAttentionMetadata:
|
||||
batch_spec = BatchSpec(seq_lens=[1] * batch_size, query_lens=[1] * batch_size)
|
||||
common_attn_metadata = create_common_attn_metadata(
|
||||
@@ -170,27 +179,18 @@ class QKNormRoPEKVCacheTestModel(torch.nn.Module):
|
||||
max_blocks = (max(batch_spec.seq_lens) + self.block_size - 1) // self.block_size
|
||||
num_blocks = batch_size * max_blocks
|
||||
|
||||
attn_backend = self.attn.attn_backend
|
||||
kv_cache_shape = attn_backend.get_kv_cache_shape(
|
||||
num_blocks, self.block_size, self.num_kv_heads, self.head_size
|
||||
)
|
||||
# Caller can force a physical layout; else use the backend's.
|
||||
if kv_stride_order is None:
|
||||
try:
|
||||
kv_stride_order = attn_backend.get_kv_cache_stride_order()
|
||||
except (AttributeError, NotImplementedError):
|
||||
kv_stride_order = tuple(range(len(kv_cache_shape)))
|
||||
|
||||
kv_cache_shape = tuple(kv_cache_shape[i] for i in kv_stride_order)
|
||||
inv_order = [kv_stride_order.index(i) for i in range(len(kv_stride_order))]
|
||||
|
||||
raw_tensor = torch.zeros(
|
||||
2 * num_blocks * self.block_size * self.num_kv_heads * self.head_size,
|
||||
dtype=self.kv_cache_dtype,
|
||||
num_blocks * self.kv_cache_spec.page_size_bytes,
|
||||
dtype=torch.int8,
|
||||
device=self.device,
|
||||
)
|
||||
raw_tensor = raw_tensor.view(kv_cache_shape)
|
||||
kv_cache = raw_tensor.permute(*inv_order)
|
||||
kv_cache = dense_kv_cache_views(
|
||||
raw_tensor,
|
||||
self.kv_cache_spec,
|
||||
num_blocks,
|
||||
num_layers=1,
|
||||
layout=layout,
|
||||
)[0]
|
||||
|
||||
# Store as a bare tensor (not wrapped in a list) to match production
|
||||
# `bind_kv_cache` behavior. `get_attention_context` returns this
|
||||
@@ -272,7 +272,7 @@ def _run_qk_norm_rope_kvcache_fusion_test(
|
||||
block_size: int,
|
||||
is_neox: bool,
|
||||
use_shuffle_kv_layout: str,
|
||||
kv_stride_order: tuple[int, ...],
|
||||
kv_layout: KVCacheLayout,
|
||||
dtype: torch.dtype,
|
||||
kv_cache_dtype: str,
|
||||
rms_norm_eps: float,
|
||||
@@ -345,7 +345,7 @@ def _run_qk_norm_rope_kvcache_fusion_test(
|
||||
# Run unfused (eager) forward
|
||||
with set_forward_context(None, vllm_config):
|
||||
forward_context = get_forward_context()
|
||||
attn_metadata = model.build_attn_metadata(num_tokens, kv_stride_order)
|
||||
attn_metadata = model.build_attn_metadata(num_tokens, kv_layout)
|
||||
forward_context.slot_mapping = {
|
||||
model.layer_name: attn_metadata.slot_mapping
|
||||
}
|
||||
@@ -360,7 +360,7 @@ def _run_qk_norm_rope_kvcache_fusion_test(
|
||||
with set_forward_context(None, vllm_config):
|
||||
model_fused = torch.compile(model, backend=backend)
|
||||
forward_context = get_forward_context()
|
||||
attn_metadata = model_fused.build_attn_metadata(num_tokens, kv_stride_order)
|
||||
attn_metadata = model_fused.build_attn_metadata(num_tokens, kv_layout)
|
||||
forward_context.slot_mapping = {
|
||||
model.layer_name: attn_metadata.slot_mapping
|
||||
}
|
||||
@@ -437,10 +437,11 @@ _FUSION_CONFIGS = [
|
||||
)
|
||||
@pytest.mark.parametrize("num_tokens", [5, 2048])
|
||||
@pytest.mark.parametrize(
|
||||
"kv_stride_order",
|
||||
# FA/unified use the 4D packed cache (num_blocks, num_kv_heads, block_size,
|
||||
# 2*head_size); no separate K/V dim to permute.
|
||||
[pytest.param((0, 1, 2, 3), id="packed_4d")],
|
||||
"kv_layout",
|
||||
[
|
||||
pytest.param(KVCacheLayout.LBHNC, id="head_major"),
|
||||
pytest.param(KVCacheLayout.LBNHC, id="token_major"),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("enable_aiter_triton_rope", [True, False])
|
||||
@pytest.mark.parametrize("block_size", [16, 32, 64])
|
||||
@@ -467,7 +468,7 @@ def test_qk_norm_rope_kvcache_fusion(
|
||||
is_neox: bool,
|
||||
attn_backend: AttentionBackendEnum,
|
||||
enable_aiter_triton_rope: bool,
|
||||
kv_stride_order: tuple[int, ...],
|
||||
kv_layout: KVCacheLayout,
|
||||
block_size: int,
|
||||
dtype: torch.dtype,
|
||||
kv_cache_dtype: str,
|
||||
@@ -486,7 +487,7 @@ def test_qk_norm_rope_kvcache_fusion(
|
||||
block_size=block_size,
|
||||
is_neox=is_neox,
|
||||
use_shuffle_kv_layout="0",
|
||||
kv_stride_order=kv_stride_order,
|
||||
kv_layout=kv_layout,
|
||||
dtype=dtype,
|
||||
kv_cache_dtype=kv_cache_dtype,
|
||||
rms_norm_eps=rms_norm_eps,
|
||||
|
||||
@@ -7,7 +7,11 @@ from torch._higher_order_ops import auto_functionalized
|
||||
|
||||
import vllm.config
|
||||
from tests.compile.backend import TestBackend
|
||||
from tests.v1.attention.utils import BatchSpec, create_common_attn_metadata
|
||||
from tests.v1.attention.utils import (
|
||||
BatchSpec,
|
||||
create_common_attn_metadata,
|
||||
dense_kv_cache_views,
|
||||
)
|
||||
from vllm._aiter_ops import is_aiter_found_and_supported, rocm_aiter_ops
|
||||
from vllm.compilation.passes.fusion import rope_kvcache_fusion
|
||||
from vllm.compilation.passes.fusion.matcher_utils import ROTARY_OP
|
||||
@@ -38,6 +42,10 @@ from vllm.v1.attention.backend import (
|
||||
CommonAttentionMetadata,
|
||||
)
|
||||
from vllm.v1.attention.backends.registry import AttentionBackendEnum
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
FullAttentionSpec,
|
||||
KVCacheLayout,
|
||||
)
|
||||
|
||||
INDEX_SELECT_OP = torch.ops.aten.index.Tensor
|
||||
VLLM_UNIFIED_KV_CACHE_UPDATE_OP = torch.ops.vllm.unified_kv_cache_update
|
||||
@@ -149,9 +157,20 @@ class QKRoPEKVCacheTestModel(torch.nn.Module):
|
||||
FP8_DTYPE if kv_cache_dtype_str.startswith("fp8") else self.dtype
|
||||
)
|
||||
|
||||
# Mirror the worker spec-collection loop: the backend publishes its
|
||||
# packing (e.g. ROCM_ATTN's separate K/V head groups).
|
||||
self.kv_cache_spec = self.attn_backend.customize_spec(
|
||||
FullAttentionSpec(
|
||||
block_size=self.block_size,
|
||||
num_kv_heads=self.num_kv_heads,
|
||||
head_size=self.head_size,
|
||||
dtype=self.kv_cache_dtype,
|
||||
)
|
||||
)
|
||||
|
||||
# Initialize attn MetadataBuilder
|
||||
self.builder = self.attn_backend.get_builder_cls()(
|
||||
kv_cache_spec=self.attn.get_kv_cache_spec(vllm_config),
|
||||
kv_cache_spec=self.kv_cache_spec,
|
||||
layer_names=[self.attn.layer_name],
|
||||
vllm_config=vllm_config,
|
||||
device=device,
|
||||
@@ -168,30 +187,25 @@ class QKRoPEKVCacheTestModel(torch.nn.Module):
|
||||
max_blocks = (max(batch_spec.seq_lens) + self.block_size - 1) // self.block_size
|
||||
num_blocks = batch_size * max_blocks
|
||||
|
||||
# Fetch the attention backend and kv cache shape and stride order
|
||||
kv_cache_shape = self.attn_backend.get_kv_cache_shape(
|
||||
num_blocks, self.block_size, self.num_kv_heads, self.head_size
|
||||
)
|
||||
try:
|
||||
kv_cache_stride_order = self.attn_backend.get_kv_cache_stride_order()
|
||||
except (AttributeError, NotImplementedError):
|
||||
kv_cache_stride_order = tuple(range(len(kv_cache_shape)))
|
||||
# A backend's preferred supported layout wins over the default, mirroring
|
||||
# selector-time resolution (e.g. ROCM_ATTN's head groups force LHBNC).
|
||||
layout = KVCacheLayout.LBNHC
|
||||
supported = self.attn_backend.supported_kv_cache_layouts()
|
||||
if supported:
|
||||
layout = supported[0]
|
||||
|
||||
kv_cache_shape = tuple(kv_cache_shape[i] for i in kv_cache_stride_order)
|
||||
inv_order = [
|
||||
kv_cache_stride_order.index(i) for i in range(len(kv_cache_stride_order))
|
||||
]
|
||||
|
||||
# Create dummy KV cache
|
||||
raw_tensor = torch.zeros(
|
||||
2 * num_blocks * self.block_size * self.num_kv_heads * self.head_size,
|
||||
dtype=self.kv_cache_dtype,
|
||||
num_blocks * self.kv_cache_spec.page_size_bytes,
|
||||
dtype=torch.int8,
|
||||
device=self.device,
|
||||
)
|
||||
raw_tensor = raw_tensor.view(kv_cache_shape)
|
||||
kv_cache = raw_tensor.permute(*inv_order)
|
||||
|
||||
self.attn.kv_cache = kv_cache
|
||||
self.attn.kv_cache = dense_kv_cache_views(
|
||||
raw_tensor,
|
||||
self.kv_cache_spec,
|
||||
num_blocks,
|
||||
num_layers=1,
|
||||
layout=layout,
|
||||
)[0]
|
||||
|
||||
# Build attn metadata
|
||||
attn_metadata = self.builder.build(
|
||||
|
||||
Reference in New Issue
Block a user