[6/N][KV-Cache Layout Refactor] Standardize KV cache layout (#51718)
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
This commit is contained in:
@@ -653,7 +653,8 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
|
||||
# per-tensor fp8 writes a separately-allocated fp8 q and quantizes the
|
||||
# KV row.
|
||||
block_size = swa_metadata.block_size
|
||||
swa_kv_cache_3d = swa_kv_cache.view(-1, block_size, self.head_dim)
|
||||
assert swa_kv_cache.shape[1:] == (block_size, self.head_dim)
|
||||
swa_kv_cache_3d = swa_kv_cache
|
||||
if cache_dtype == torch.bfloat16:
|
||||
torch.ops._C.fused_deepseek_v4_qnorm_rope_kv_rope_full_cache_bf16_insert(
|
||||
q,
|
||||
@@ -684,6 +685,17 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
|
||||
)
|
||||
return q_fp8
|
||||
|
||||
def _global_topk_output_buffers(
|
||||
self, topk_indices: torch.Tensor
|
||||
) -> tuple[torch.Tensor, torch.Tensor] | None:
|
||||
if self.compress_ratio != 4 or self.eager_scratch_pool is None:
|
||||
return None
|
||||
return self.eager_scratch_pool.global_topk_outputs(topk_indices)
|
||||
|
||||
def bind_kv_cache(self, kv_cache: torch.Tensor) -> None:
|
||||
# [B, H=1, N, C] -> [B, N, C]
|
||||
self.kv_cache = kv_cache.squeeze(1)
|
||||
|
||||
def get_attn_backend(self) -> type[AttentionBackend]:
|
||||
return self.backend_cls
|
||||
|
||||
@@ -701,7 +713,7 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
|
||||
num_kv_heads=1,
|
||||
head_size=self.head_dim,
|
||||
dtype=torch.uint8 if uses_fp8_ds_mla_layout else self.kv_cache_torch_dtype,
|
||||
compress_ratio=self.compress_ratio,
|
||||
tokens_per_state=self.compress_ratio,
|
||||
cache_dtype_str=self.kv_cache_dtype,
|
||||
alignment=576 if uses_fp8_ds_mla_layout else 512,
|
||||
model_version="deepseek_v4",
|
||||
@@ -733,16 +745,20 @@ class DeepseekV4IndexerCache(torch.nn.Module, AttentionLayerBase):
|
||||
raise ValueError(f"Duplicate layer name: {prefix}")
|
||||
compilation_config.static_forward_context[prefix] = self
|
||||
|
||||
def bind_kv_cache(self, kv_cache: torch.Tensor) -> None:
|
||||
# [B, H=1, N, C] -> [B, N, C]
|
||||
self.kv_cache = kv_cache.squeeze(1)
|
||||
|
||||
def get_kv_cache_spec(self, vllm_config: VllmConfig) -> KVCacheSpec:
|
||||
# head_dim already carries the fp8 scale padding
|
||||
# compress_ratio=1 for V3.2, >1 for DeepseekV4; both use the same cache layout.
|
||||
# tokens_per_state=1 for V3.2, >1 for DeepseekV4; same cache layout.
|
||||
uses_fp8_ds_mla_layout = vllm_config.cache_config.cache_dtype == "fp8_ds_mla"
|
||||
return MLAAttentionSpec(
|
||||
block_size=self.cache_config.block_size,
|
||||
num_kv_heads=1,
|
||||
head_size=self.head_dim,
|
||||
dtype=self.dtype,
|
||||
compress_ratio=self.compress_ratio,
|
||||
tokens_per_state=self.compress_ratio,
|
||||
# 576B for FlashMLA packing; 512B for FlashInfer sparse (#44577).
|
||||
alignment=576 if uses_fp8_ds_mla_layout else 512,
|
||||
)
|
||||
|
||||
@@ -75,25 +75,6 @@ class CompressorBackend(AttentionBackend):
|
||||
def get_builder_cls() -> type["CompressorMetadataBuilder"]:
|
||||
return CompressorMetadataBuilder
|
||||
|
||||
@staticmethod
|
||||
def get_kv_cache_shape(
|
||||
num_blocks: int,
|
||||
block_size: int,
|
||||
num_kv_heads: int,
|
||||
head_size: int,
|
||||
cache_dtype_str: str = "auto",
|
||||
) -> tuple[int, ...]:
|
||||
assert num_kv_heads == 1
|
||||
return (num_blocks, block_size, head_size)
|
||||
|
||||
@staticmethod
|
||||
def get_kv_cache_stride_order(
|
||||
include_num_layers_dimension: bool = False,
|
||||
) -> tuple[int, ...]:
|
||||
if include_num_layers_dimension:
|
||||
return (0, 1, 2, 3)
|
||||
return (0, 1, 2)
|
||||
|
||||
|
||||
@dataclass
|
||||
class CompressorMetadata:
|
||||
@@ -185,6 +166,10 @@ class CompressorStateCache(torch.nn.Module, AttentionLayerBase):
|
||||
else:
|
||||
raise ValueError(f"Invalid compress ratio: {compress_ratio}")
|
||||
|
||||
def bind_kv_cache(self, kv_cache: torch.Tensor) -> None:
|
||||
# [B, H=1, N, C] -> [B, N, C]
|
||||
self.kv_cache = kv_cache.squeeze(1)
|
||||
|
||||
def get_kv_cache_spec(self, vllm_config: VllmConfig) -> KVCacheSpec:
|
||||
# fp8_ds_mla is the UE8M0 paged layout and needs 576B alignment. Plain
|
||||
# full-cache rows share state pages with contiguous KV pages, so padding
|
||||
|
||||
@@ -22,7 +22,6 @@ from vllm.models.deepseek_v4.sparse_mla import (
|
||||
DeepseekV4FlashMLAMetadata,
|
||||
DeepseekV4SparseMLABackend,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.platforms.interface import DeviceCapability
|
||||
from vllm.utils.flashinfer import flashinfer_trtllm_batch_decode_sparse_mla_dsv4
|
||||
from vllm.v1.attention.backend import MultipleOf
|
||||
@@ -165,26 +164,6 @@ class DeepseekV4FlashInferMLASparseBackend(DeepseekV4SparseMLABackend):
|
||||
return None
|
||||
return "FLASHINFER_MLA_SPARSE_DSV4 requires SM10x or SM12x"
|
||||
|
||||
@staticmethod
|
||||
def get_kv_cache_shape(
|
||||
num_blocks: int,
|
||||
block_size: int,
|
||||
num_kv_heads: int,
|
||||
head_size: int,
|
||||
cache_dtype_str: str = "auto",
|
||||
) -> tuple[int, ...]:
|
||||
device_capability = current_platform.get_device_capability()
|
||||
if device_capability is not None and device_capability.major == 12:
|
||||
return DeepseekV4SparseMLABackend.get_kv_cache_shape(
|
||||
num_blocks,
|
||||
block_size,
|
||||
num_kv_heads,
|
||||
head_size,
|
||||
cache_dtype_str,
|
||||
)
|
||||
assert num_kv_heads == 1
|
||||
return (num_blocks, block_size, head_size)
|
||||
|
||||
|
||||
class DeepseekV4FlashInferMLAAttention(DeepseekV4Attention):
|
||||
"""FlashInfer TRTLLM-gen sparse MLA attention layer for SM100 DeepSeek V4."""
|
||||
|
||||
@@ -88,21 +88,6 @@ class DeepseekV4SparseMLABackend(AttentionBackend):
|
||||
def supports_compute_capability(cls, capability: DeviceCapability) -> bool:
|
||||
return capability.major in [9, 10]
|
||||
|
||||
@staticmethod
|
||||
def get_kv_cache_shape(
|
||||
num_blocks: int,
|
||||
block_size: int,
|
||||
num_kv_heads: int,
|
||||
head_size: int,
|
||||
cache_dtype_str: str = "auto",
|
||||
) -> tuple[int, ...]:
|
||||
if cache_dtype_str == "fp8_ds_mla":
|
||||
# DeepseekV4 main MLA: 584B per token (448 NoPE + 128 RoPE + 8 fp8 scale).
|
||||
# head_size passed in is the semantic head_dim (512).
|
||||
return (num_blocks, block_size, 584)
|
||||
else:
|
||||
return (num_blocks, block_size, head_size)
|
||||
|
||||
|
||||
@dataclass
|
||||
class DeepseekV4FlashMLAMetadata(AttentionMetadata):
|
||||
@@ -151,8 +136,8 @@ class DeepseekV4SparseMLAMetadataBuilder(
|
||||
(max_num_batched_tokens,), dtype=torch.int32, device=device
|
||||
)
|
||||
|
||||
assert hasattr(self.kv_cache_spec, "compress_ratio")
|
||||
self.compress_ratio = self.kv_cache_spec.compress_ratio
|
||||
assert isinstance(self.kv_cache_spec.tokens_per_state, int)
|
||||
self.compress_ratio = self.kv_cache_spec.tokens_per_state
|
||||
|
||||
# Pre-allocate compressed slot mapping buffer for CUDA graph address
|
||||
# stability when compress_ratio > 1.
|
||||
@@ -206,7 +191,7 @@ class DeepseekV4SparseMLAMetadataBuilder(
|
||||
cm.query_start_loc,
|
||||
cm.seq_lens,
|
||||
cm.block_table_tensor.clamp_(min=0),
|
||||
int(self.kv_cache_spec.storage_block_size),
|
||||
int(self.kv_cache_spec.num_states),
|
||||
self.compress_ratio,
|
||||
out=self.compressed_slot_mapping_buffer,
|
||||
)
|
||||
|
||||
@@ -107,32 +107,6 @@ class InklingSconvBackend(AttentionBackend):
|
||||
def get_name() -> str:
|
||||
return "INKLING_SCONV_SWA"
|
||||
|
||||
@classmethod
|
||||
def indexes_kv_by_block_stride(cls) -> bool:
|
||||
# num_blocks is the outermost dim (HND, see get_kv_cache_shape), so the
|
||||
# padded conv page is read through a strided view.
|
||||
return True
|
||||
|
||||
@staticmethod
|
||||
def get_kv_cache_shape(
|
||||
num_blocks: int,
|
||||
block_size: int,
|
||||
num_kv_heads: int,
|
||||
head_size: int,
|
||||
cache_dtype_str: str = "auto",
|
||||
) -> tuple[int, ...]:
|
||||
# HND, num-blocks-first, head-major: [num_blocks, H, N, D].
|
||||
return (num_blocks, num_kv_heads, block_size, head_size)
|
||||
|
||||
@staticmethod
|
||||
def get_kv_cache_stride_order(
|
||||
include_num_layers_dimension: bool = False,
|
||||
) -> tuple[int, ...]:
|
||||
# Identity: physical layout == logical [num_blocks, H, N, D].
|
||||
if include_num_layers_dimension:
|
||||
return (0, 1, 2, 3, 4)
|
||||
return (0, 1, 2, 3)
|
||||
|
||||
@staticmethod
|
||||
def get_impl_cls():
|
||||
raise NotImplementedError(
|
||||
|
||||
@@ -107,32 +107,6 @@ class InklingSconvBackend(AttentionBackend):
|
||||
def get_name() -> str:
|
||||
return "INKLING_SCONV_SWA"
|
||||
|
||||
@classmethod
|
||||
def indexes_kv_by_block_stride(cls) -> bool:
|
||||
# num_blocks is the outermost dim (HND, see get_kv_cache_shape), so the
|
||||
# padded conv page is read through a strided view.
|
||||
return True
|
||||
|
||||
@staticmethod
|
||||
def get_kv_cache_shape(
|
||||
num_blocks: int,
|
||||
block_size: int,
|
||||
num_kv_heads: int,
|
||||
head_size: int,
|
||||
cache_dtype_str: str = "auto",
|
||||
) -> tuple[int, ...]:
|
||||
# HND, num-blocks-first, head-major: [num_blocks, H, N, D].
|
||||
return (num_blocks, num_kv_heads, block_size, head_size)
|
||||
|
||||
@staticmethod
|
||||
def get_kv_cache_stride_order(
|
||||
include_num_layers_dimension: bool = False,
|
||||
) -> tuple[int, ...]:
|
||||
# Identity: physical layout == logical [num_blocks, H, N, D].
|
||||
if include_num_layers_dimension:
|
||||
return (0, 1, 2, 3, 4)
|
||||
return (0, 1, 2, 3)
|
||||
|
||||
@staticmethod
|
||||
def get_impl_cls():
|
||||
raise NotImplementedError(
|
||||
|
||||
@@ -384,6 +384,10 @@ class MultiHeadLatentAttention(nn.Module, AttentionLayerBase):
|
||||
# ------------------------------------------------------------------
|
||||
# AttentionLayerBase interface
|
||||
# ------------------------------------------------------------------
|
||||
def bind_kv_cache(self, kv_cache: torch.Tensor) -> None:
|
||||
# [B, H=1, N, C] -> [B, N, C]
|
||||
self.kv_cache = kv_cache.squeeze(1)
|
||||
|
||||
def get_attn_backend(self) -> type[AttentionBackend]:
|
||||
return self.attn_backend
|
||||
|
||||
|
||||
@@ -728,6 +728,12 @@ class MiniMaxM3SparseAttention(nn.Module, AttentionLayerBase):
|
||||
|
||||
def get_kv_cache_spec(self, vllm_config: VllmConfig) -> KVCacheSpec | None:
|
||||
# Main GQA K/V cache. Block size may change after load, refresh it.
|
||||
# AITER sparse PA wants K and V as two separate head groups
|
||||
# (H folded into the content dim).
|
||||
sparse_pa = self.use_aiter_sparse_pa
|
||||
kv_bytes = (
|
||||
self.num_kv_heads * self.head_dim * self.kv_cache_torch_dtype.itemsize
|
||||
)
|
||||
return FullAttentionSpec(
|
||||
block_size=vllm_config.cache_config.block_size,
|
||||
num_kv_heads=self.num_kv_heads,
|
||||
@@ -735,6 +741,8 @@ class MiniMaxM3SparseAttention(nn.Module, AttentionLayerBase):
|
||||
head_size_v=self.head_dim,
|
||||
dtype=self.kv_cache_torch_dtype,
|
||||
kv_quant_mode=get_kv_quant_mode(self.kv_cache_dtype),
|
||||
num_head_slots=2 if sparse_pa else None,
|
||||
state_content_bytes=kv_bytes if sparse_pa else None,
|
||||
)
|
||||
|
||||
def _ensure_aiter_sparse_pa_kv_cache(self) -> None:
|
||||
|
||||
@@ -97,25 +97,6 @@ class MiniMaxM3IndexerBackend(AttentionBackend):
|
||||
def is_sparse(cls) -> bool:
|
||||
return True
|
||||
|
||||
@staticmethod
|
||||
def get_kv_cache_shape(
|
||||
num_blocks: int,
|
||||
block_size: int,
|
||||
num_kv_heads: int,
|
||||
head_size: int,
|
||||
cache_dtype_str: str = "auto",
|
||||
) -> tuple[int, ...]:
|
||||
return (num_blocks, block_size, head_size)
|
||||
|
||||
@staticmethod
|
||||
def get_kv_cache_stride_order(
|
||||
include_num_layers_dimension: bool = False,
|
||||
) -> tuple[int, ...]:
|
||||
if include_num_layers_dimension:
|
||||
# M3 does not use cross-layer (per-layer-stacked) KV blocks.
|
||||
raise NotImplementedError
|
||||
return (0, 1, 2)
|
||||
|
||||
|
||||
class MiniMaxM3IndexerCache(nn.Module, AttentionLayerBase):
|
||||
"""Side KV cache for the indexer's per-token index keys (key-only).
|
||||
@@ -156,6 +137,10 @@ class MiniMaxM3IndexerCache(nn.Module, AttentionLayerBase):
|
||||
raise ValueError(f"Duplicate layer name: {prefix}")
|
||||
compilation_config.static_forward_context[prefix] = self
|
||||
|
||||
def bind_kv_cache(self, kv_cache: torch.Tensor) -> None:
|
||||
# [B, H=1, N, C] -> [B, N, C]
|
||||
self.kv_cache = kv_cache.squeeze(1)
|
||||
|
||||
def get_kv_cache_spec(self, vllm_config: VllmConfig) -> KVCacheSpec:
|
||||
# Key-only: MLAAttentionSpec budgets one vector/token (not 2x for K+V).
|
||||
return MLAAttentionSpec(
|
||||
|
||||
@@ -51,10 +51,13 @@ from vllm.v1.attention.backend import (
|
||||
MultipleOf,
|
||||
)
|
||||
from vllm.v1.attention.backends.utils import (
|
||||
get_kv_cache_layout,
|
||||
split_decodes_and_prefills,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import AttentionSpec, is_quantized_kv_cache
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
AttentionSpec,
|
||||
KVCacheLayout,
|
||||
is_quantized_kv_cache,
|
||||
)
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -112,47 +115,13 @@ class MiniMaxM3SparseBackend(AttentionBackend):
|
||||
def is_sparse(cls) -> bool:
|
||||
return True
|
||||
|
||||
@staticmethod
|
||||
def get_kv_cache_shape(
|
||||
num_blocks: int,
|
||||
block_size: int,
|
||||
num_kv_heads: int,
|
||||
head_size: int,
|
||||
cache_dtype_str: str = "auto",
|
||||
) -> tuple[int, ...]:
|
||||
if minimax_m3_use_aiter_sparse_pa(num_kv_heads):
|
||||
# AITER's assembly paged-attention kernels require independently
|
||||
# contiguous K and V storage. Keep that specialized layout behind
|
||||
# the shuffle flag while every other implementation uses the
|
||||
# packed-content contract introduced by #44455.
|
||||
return (num_blocks, 2, block_size, num_kv_heads, head_size)
|
||||
# K and V are packed into the content dim: logical (B, H, N, 2*hs).
|
||||
return (num_blocks, num_kv_heads, block_size, 2 * head_size)
|
||||
|
||||
@staticmethod
|
||||
def get_kv_cache_stride_order(
|
||||
include_num_layers_dimension: bool = False,
|
||||
) -> tuple[int, ...]:
|
||||
# `stride_order` indicates the permutation that gets us from
|
||||
# `get_kv_cache_shape` (logical (B, H, N, 2*hs)) to the actual memory
|
||||
# layout we want.
|
||||
if include_num_layers_dimension:
|
||||
raise NotImplementedError # no cross-layer KV blocks in M3
|
||||
@classmethod
|
||||
def supported_kv_cache_layouts(cls) -> tuple[KVCacheLayout, ...] | None:
|
||||
# AITER sparse PA unbinds K and V as separate contiguous regions, which
|
||||
# requires the head dimension to sit outside the block dimension.
|
||||
if _minimax_m3_aiter_sparse_pa_requested():
|
||||
# The AITER page-16 sparse PA path reinterprets the K and V slices
|
||||
# as separate SHUFFLE caches. Keep K/V physically separated so
|
||||
# kv_cache[:, 0] and kv_cache[:, 1] are contiguous byte ranges.
|
||||
return (1, 0, 2, 3, 4)
|
||||
cache_layout = get_kv_cache_layout()
|
||||
if cache_layout == "NHD":
|
||||
# (num_blocks, block_size, num_kv_heads, 2*head_size)
|
||||
stride_order = (0, 2, 1, 3)
|
||||
elif cache_layout == "HND":
|
||||
# (num_blocks, num_kv_heads, block_size, 2*head_size)
|
||||
stride_order = (0, 1, 2, 3)
|
||||
else:
|
||||
raise ValueError(f"Unknown cache layout format {cache_layout}.")
|
||||
return stride_order
|
||||
return (KVCacheLayout.LHBNC,)
|
||||
return super().supported_kv_cache_layouts()
|
||||
|
||||
|
||||
@dataclass
|
||||
|
||||
Reference in New Issue
Block a user