[6/N][KV-Cache Layout Refactor] Standardize KV cache layout (#51718)

Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
This commit is contained in:
Lucas Wilkinson
2026-08-21 17:47:49 -07:00
committed by GitHub
parent 9ff7041b55
commit 8bdc70ec7b
150 changed files with 4006 additions and 5815 deletions
+20 -4
View File
@@ -653,7 +653,8 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
# per-tensor fp8 writes a separately-allocated fp8 q and quantizes the
# KV row.
block_size = swa_metadata.block_size
swa_kv_cache_3d = swa_kv_cache.view(-1, block_size, self.head_dim)
assert swa_kv_cache.shape[1:] == (block_size, self.head_dim)
swa_kv_cache_3d = swa_kv_cache
if cache_dtype == torch.bfloat16:
torch.ops._C.fused_deepseek_v4_qnorm_rope_kv_rope_full_cache_bf16_insert(
q,
@@ -684,6 +685,17 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
)
return q_fp8
def _global_topk_output_buffers(
self, topk_indices: torch.Tensor
) -> tuple[torch.Tensor, torch.Tensor] | None:
if self.compress_ratio != 4 or self.eager_scratch_pool is None:
return None
return self.eager_scratch_pool.global_topk_outputs(topk_indices)
def bind_kv_cache(self, kv_cache: torch.Tensor) -> None:
# [B, H=1, N, C] -> [B, N, C]
self.kv_cache = kv_cache.squeeze(1)
def get_attn_backend(self) -> type[AttentionBackend]:
return self.backend_cls
@@ -701,7 +713,7 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
num_kv_heads=1,
head_size=self.head_dim,
dtype=torch.uint8 if uses_fp8_ds_mla_layout else self.kv_cache_torch_dtype,
compress_ratio=self.compress_ratio,
tokens_per_state=self.compress_ratio,
cache_dtype_str=self.kv_cache_dtype,
alignment=576 if uses_fp8_ds_mla_layout else 512,
model_version="deepseek_v4",
@@ -733,16 +745,20 @@ class DeepseekV4IndexerCache(torch.nn.Module, AttentionLayerBase):
raise ValueError(f"Duplicate layer name: {prefix}")
compilation_config.static_forward_context[prefix] = self
def bind_kv_cache(self, kv_cache: torch.Tensor) -> None:
# [B, H=1, N, C] -> [B, N, C]
self.kv_cache = kv_cache.squeeze(1)
def get_kv_cache_spec(self, vllm_config: VllmConfig) -> KVCacheSpec:
# head_dim already carries the fp8 scale padding
# compress_ratio=1 for V3.2, >1 for DeepseekV4; both use the same cache layout.
# tokens_per_state=1 for V3.2, >1 for DeepseekV4; same cache layout.
uses_fp8_ds_mla_layout = vllm_config.cache_config.cache_dtype == "fp8_ds_mla"
return MLAAttentionSpec(
block_size=self.cache_config.block_size,
num_kv_heads=1,
head_size=self.head_dim,
dtype=self.dtype,
compress_ratio=self.compress_ratio,
tokens_per_state=self.compress_ratio,
# 576B for FlashMLA packing; 512B for FlashInfer sparse (#44577).
alignment=576 if uses_fp8_ds_mla_layout else 512,
)
+4 -19
View File
@@ -75,25 +75,6 @@ class CompressorBackend(AttentionBackend):
def get_builder_cls() -> type["CompressorMetadataBuilder"]:
return CompressorMetadataBuilder
@staticmethod
def get_kv_cache_shape(
num_blocks: int,
block_size: int,
num_kv_heads: int,
head_size: int,
cache_dtype_str: str = "auto",
) -> tuple[int, ...]:
assert num_kv_heads == 1
return (num_blocks, block_size, head_size)
@staticmethod
def get_kv_cache_stride_order(
include_num_layers_dimension: bool = False,
) -> tuple[int, ...]:
if include_num_layers_dimension:
return (0, 1, 2, 3)
return (0, 1, 2)
@dataclass
class CompressorMetadata:
@@ -185,6 +166,10 @@ class CompressorStateCache(torch.nn.Module, AttentionLayerBase):
else:
raise ValueError(f"Invalid compress ratio: {compress_ratio}")
def bind_kv_cache(self, kv_cache: torch.Tensor) -> None:
# [B, H=1, N, C] -> [B, N, C]
self.kv_cache = kv_cache.squeeze(1)
def get_kv_cache_spec(self, vllm_config: VllmConfig) -> KVCacheSpec:
# fp8_ds_mla is the UE8M0 paged layout and needs 576B alignment. Plain
# full-cache rows share state pages with contiguous KV pages, so padding
@@ -22,7 +22,6 @@ from vllm.models.deepseek_v4.sparse_mla import (
DeepseekV4FlashMLAMetadata,
DeepseekV4SparseMLABackend,
)
from vllm.platforms import current_platform
from vllm.platforms.interface import DeviceCapability
from vllm.utils.flashinfer import flashinfer_trtllm_batch_decode_sparse_mla_dsv4
from vllm.v1.attention.backend import MultipleOf
@@ -165,26 +164,6 @@ class DeepseekV4FlashInferMLASparseBackend(DeepseekV4SparseMLABackend):
return None
return "FLASHINFER_MLA_SPARSE_DSV4 requires SM10x or SM12x"
@staticmethod
def get_kv_cache_shape(
num_blocks: int,
block_size: int,
num_kv_heads: int,
head_size: int,
cache_dtype_str: str = "auto",
) -> tuple[int, ...]:
device_capability = current_platform.get_device_capability()
if device_capability is not None and device_capability.major == 12:
return DeepseekV4SparseMLABackend.get_kv_cache_shape(
num_blocks,
block_size,
num_kv_heads,
head_size,
cache_dtype_str,
)
assert num_kv_heads == 1
return (num_blocks, block_size, head_size)
class DeepseekV4FlashInferMLAAttention(DeepseekV4Attention):
"""FlashInfer TRTLLM-gen sparse MLA attention layer for SM100 DeepSeek V4."""
+3 -18
View File
@@ -88,21 +88,6 @@ class DeepseekV4SparseMLABackend(AttentionBackend):
def supports_compute_capability(cls, capability: DeviceCapability) -> bool:
return capability.major in [9, 10]
@staticmethod
def get_kv_cache_shape(
num_blocks: int,
block_size: int,
num_kv_heads: int,
head_size: int,
cache_dtype_str: str = "auto",
) -> tuple[int, ...]:
if cache_dtype_str == "fp8_ds_mla":
# DeepseekV4 main MLA: 584B per token (448 NoPE + 128 RoPE + 8 fp8 scale).
# head_size passed in is the semantic head_dim (512).
return (num_blocks, block_size, 584)
else:
return (num_blocks, block_size, head_size)
@dataclass
class DeepseekV4FlashMLAMetadata(AttentionMetadata):
@@ -151,8 +136,8 @@ class DeepseekV4SparseMLAMetadataBuilder(
(max_num_batched_tokens,), dtype=torch.int32, device=device
)
assert hasattr(self.kv_cache_spec, "compress_ratio")
self.compress_ratio = self.kv_cache_spec.compress_ratio
assert isinstance(self.kv_cache_spec.tokens_per_state, int)
self.compress_ratio = self.kv_cache_spec.tokens_per_state
# Pre-allocate compressed slot mapping buffer for CUDA graph address
# stability when compress_ratio > 1.
@@ -206,7 +191,7 @@ class DeepseekV4SparseMLAMetadataBuilder(
cm.query_start_loc,
cm.seq_lens,
cm.block_table_tensor.clamp_(min=0),
int(self.kv_cache_spec.storage_block_size),
int(self.kv_cache_spec.num_states),
self.compress_ratio,
out=self.compressed_slot_mapping_buffer,
)
-26
View File
@@ -107,32 +107,6 @@ class InklingSconvBackend(AttentionBackend):
def get_name() -> str:
return "INKLING_SCONV_SWA"
@classmethod
def indexes_kv_by_block_stride(cls) -> bool:
# num_blocks is the outermost dim (HND, see get_kv_cache_shape), so the
# padded conv page is read through a strided view.
return True
@staticmethod
def get_kv_cache_shape(
num_blocks: int,
block_size: int,
num_kv_heads: int,
head_size: int,
cache_dtype_str: str = "auto",
) -> tuple[int, ...]:
# HND, num-blocks-first, head-major: [num_blocks, H, N, D].
return (num_blocks, num_kv_heads, block_size, head_size)
@staticmethod
def get_kv_cache_stride_order(
include_num_layers_dimension: bool = False,
) -> tuple[int, ...]:
# Identity: physical layout == logical [num_blocks, H, N, D].
if include_num_layers_dimension:
return (0, 1, 2, 3, 4)
return (0, 1, 2, 3)
@staticmethod
def get_impl_cls():
raise NotImplementedError(
@@ -107,32 +107,6 @@ class InklingSconvBackend(AttentionBackend):
def get_name() -> str:
return "INKLING_SCONV_SWA"
@classmethod
def indexes_kv_by_block_stride(cls) -> bool:
# num_blocks is the outermost dim (HND, see get_kv_cache_shape), so the
# padded conv page is read through a strided view.
return True
@staticmethod
def get_kv_cache_shape(
num_blocks: int,
block_size: int,
num_kv_heads: int,
head_size: int,
cache_dtype_str: str = "auto",
) -> tuple[int, ...]:
# HND, num-blocks-first, head-major: [num_blocks, H, N, D].
return (num_blocks, num_kv_heads, block_size, head_size)
@staticmethod
def get_kv_cache_stride_order(
include_num_layers_dimension: bool = False,
) -> tuple[int, ...]:
# Identity: physical layout == logical [num_blocks, H, N, D].
if include_num_layers_dimension:
return (0, 1, 2, 3, 4)
return (0, 1, 2, 3)
@staticmethod
def get_impl_cls():
raise NotImplementedError(
+4
View File
@@ -384,6 +384,10 @@ class MultiHeadLatentAttention(nn.Module, AttentionLayerBase):
# ------------------------------------------------------------------
# AttentionLayerBase interface
# ------------------------------------------------------------------
def bind_kv_cache(self, kv_cache: torch.Tensor) -> None:
# [B, H=1, N, C] -> [B, N, C]
self.kv_cache = kv_cache.squeeze(1)
def get_attn_backend(self) -> type[AttentionBackend]:
return self.attn_backend
+8
View File
@@ -728,6 +728,12 @@ class MiniMaxM3SparseAttention(nn.Module, AttentionLayerBase):
def get_kv_cache_spec(self, vllm_config: VllmConfig) -> KVCacheSpec | None:
# Main GQA K/V cache. Block size may change after load, refresh it.
# AITER sparse PA wants K and V as two separate head groups
# (H folded into the content dim).
sparse_pa = self.use_aiter_sparse_pa
kv_bytes = (
self.num_kv_heads * self.head_dim * self.kv_cache_torch_dtype.itemsize
)
return FullAttentionSpec(
block_size=vllm_config.cache_config.block_size,
num_kv_heads=self.num_kv_heads,
@@ -735,6 +741,8 @@ class MiniMaxM3SparseAttention(nn.Module, AttentionLayerBase):
head_size_v=self.head_dim,
dtype=self.kv_cache_torch_dtype,
kv_quant_mode=get_kv_quant_mode(self.kv_cache_dtype),
num_head_slots=2 if sparse_pa else None,
state_content_bytes=kv_bytes if sparse_pa else None,
)
def _ensure_aiter_sparse_pa_kv_cache(self) -> None:
+4 -19
View File
@@ -97,25 +97,6 @@ class MiniMaxM3IndexerBackend(AttentionBackend):
def is_sparse(cls) -> bool:
return True
@staticmethod
def get_kv_cache_shape(
num_blocks: int,
block_size: int,
num_kv_heads: int,
head_size: int,
cache_dtype_str: str = "auto",
) -> tuple[int, ...]:
return (num_blocks, block_size, head_size)
@staticmethod
def get_kv_cache_stride_order(
include_num_layers_dimension: bool = False,
) -> tuple[int, ...]:
if include_num_layers_dimension:
# M3 does not use cross-layer (per-layer-stacked) KV blocks.
raise NotImplementedError
return (0, 1, 2)
class MiniMaxM3IndexerCache(nn.Module, AttentionLayerBase):
"""Side KV cache for the indexer's per-token index keys (key-only).
@@ -156,6 +137,10 @@ class MiniMaxM3IndexerCache(nn.Module, AttentionLayerBase):
raise ValueError(f"Duplicate layer name: {prefix}")
compilation_config.static_forward_context[prefix] = self
def bind_kv_cache(self, kv_cache: torch.Tensor) -> None:
# [B, H=1, N, C] -> [B, N, C]
self.kv_cache = kv_cache.squeeze(1)
def get_kv_cache_spec(self, vllm_config: VllmConfig) -> KVCacheSpec:
# Key-only: MLAAttentionSpec budgets one vector/token (not 2x for K+V).
return MLAAttentionSpec(
@@ -51,10 +51,13 @@ from vllm.v1.attention.backend import (
MultipleOf,
)
from vllm.v1.attention.backends.utils import (
get_kv_cache_layout,
split_decodes_and_prefills,
)
from vllm.v1.kv_cache_interface import AttentionSpec, is_quantized_kv_cache
from vllm.v1.kv_cache_interface import (
AttentionSpec,
KVCacheLayout,
is_quantized_kv_cache,
)
logger = init_logger(__name__)
@@ -112,47 +115,13 @@ class MiniMaxM3SparseBackend(AttentionBackend):
def is_sparse(cls) -> bool:
return True
@staticmethod
def get_kv_cache_shape(
num_blocks: int,
block_size: int,
num_kv_heads: int,
head_size: int,
cache_dtype_str: str = "auto",
) -> tuple[int, ...]:
if minimax_m3_use_aiter_sparse_pa(num_kv_heads):
# AITER's assembly paged-attention kernels require independently
# contiguous K and V storage. Keep that specialized layout behind
# the shuffle flag while every other implementation uses the
# packed-content contract introduced by #44455.
return (num_blocks, 2, block_size, num_kv_heads, head_size)
# K and V are packed into the content dim: logical (B, H, N, 2*hs).
return (num_blocks, num_kv_heads, block_size, 2 * head_size)
@staticmethod
def get_kv_cache_stride_order(
include_num_layers_dimension: bool = False,
) -> tuple[int, ...]:
# `stride_order` indicates the permutation that gets us from
# `get_kv_cache_shape` (logical (B, H, N, 2*hs)) to the actual memory
# layout we want.
if include_num_layers_dimension:
raise NotImplementedError # no cross-layer KV blocks in M3
@classmethod
def supported_kv_cache_layouts(cls) -> tuple[KVCacheLayout, ...] | None:
# AITER sparse PA unbinds K and V as separate contiguous regions, which
# requires the head dimension to sit outside the block dimension.
if _minimax_m3_aiter_sparse_pa_requested():
# The AITER page-16 sparse PA path reinterprets the K and V slices
# as separate SHUFFLE caches. Keep K/V physically separated so
# kv_cache[:, 0] and kv_cache[:, 1] are contiguous byte ranges.
return (1, 0, 2, 3, 4)
cache_layout = get_kv_cache_layout()
if cache_layout == "NHD":
# (num_blocks, block_size, num_kv_heads, 2*head_size)
stride_order = (0, 2, 1, 3)
elif cache_layout == "HND":
# (num_blocks, num_kv_heads, block_size, 2*head_size)
stride_order = (0, 1, 2, 3)
else:
raise ValueError(f"Unknown cache layout format {cache_layout}.")
return stride_order
return (KVCacheLayout.LHBNC,)
return super().supported_kv_cache_layouts()
@dataclass