[6/N][KV-Cache Layout Refactor] Standardize KV cache layout (#51718)

Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
This commit is contained in:
Lucas Wilkinson
2026-08-21 17:47:49 -07:00
committed by GitHub
parent 9ff7041b55
commit 8bdc70ec7b
150 changed files with 4006 additions and 5815 deletions
+202 -40
View File
@@ -8,6 +8,7 @@ from collections import Counter
from collections.abc import Collection
from dataclasses import dataclass, fields, replace
from enum import Enum, IntEnum
from fractions import Fraction
from math import prod
from typing import TYPE_CHECKING, TypeVar
@@ -18,6 +19,7 @@ from vllm.logger import init_logger
from vllm.utils.math_utils import cdiv, round_up
from vllm.utils.torch_utils import get_dtype_size
from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum
from vllm.v1.kv_cache_layout import _DIM_B, _DIM_L, KVCacheLayout
from vllm.v1.kv_cache_spec_registry import KVCacheSpecRegistry
if TYPE_CHECKING:
@@ -147,6 +149,18 @@ class KVCacheSpec:
# number of tokens in a block
block_size: int
@property
def num_heads(self) -> int:
raise NotImplementedError
@property
def tokens_per_state(self) -> int | Fraction:
raise NotImplementedError
@property
def state_content_size_bytes(self) -> int:
raise NotImplementedError
@property
def page_size_bytes(self) -> int:
"""
@@ -158,8 +172,13 @@ class KVCacheSpec:
raise NotImplementedError
@property
def storage_block_size(self) -> int:
return self.block_size
def num_states(self) -> int:
return self.get_num_kernel_states(self.block_size)
def get_num_kernel_states(self, kernel_block_size: int) -> int:
if self.tokens_per_state > 0:
return kernel_block_size // self.tokens_per_state
return 1
def max_memory_usage_bytes(self, vllm_config: VllmConfig) -> int:
"""
@@ -214,6 +233,143 @@ class KVCacheSpec:
)
def group_kernel_blocks(cache: torch.Tensor, num_blocks: int) -> torch.Tensor:
"""View a kernel-block-granular layer cache with manager blocks as dim 0.
Kernel block splitting subdivides each manager block into uniformly strided
kernel blocks, so grouping is a pure view: ``(num_blocks * ratio, ...)``
"""
if cache.shape[0] == num_blocks:
return cache
assert cache.shape[0] % num_blocks == 0
return cache.unflatten(0, (num_blocks, -1))
def compute_layer_kv_cache_shape_bytes(
spec: KVCacheSpec,
num_blocks: int,
kernel_block_size: int | None = None,
) -> tuple[int, ...]:
"""Return the 4D logical shape ``(B, H, N, C)`` where C is in bytes."""
bs = kernel_block_size if kernel_block_size is not None else spec.block_size
assert spec.block_size % bs == 0, (
f"Kernel block size {bs} must divide KV cache block size {spec.block_size}."
)
blocks_per_page = spec.block_size // bs
return (
num_blocks * blocks_per_page,
spec.num_heads,
spec.get_num_kernel_states(bs),
spec.state_content_size_bytes,
)
def compute_layout_strides(
spec: KVCacheSpec,
num_blocks: int,
num_layers: int,
layout: KVCacheLayout,
kernel_block_size: int | None = None,
fixed_strides: tuple[int | None, ...] = (None,) * 5,
) -> tuple[int, ...]:
"""Byte strides in logical ``[L, B, H, N, C]`` axis order."""
assert len(fixed_strides) == 5
assert all(stride is None or stride > 0 for stride in fixed_strides)
shape = (
num_layers,
*compute_layer_kv_cache_shape_bytes(spec, num_blocks, kernel_block_size),
)
order = layout.stride_order
padded_page_size = getattr(spec, "page_size_padded", None)
if padded_page_size is not None:
assert kernel_block_size is None or kernel_block_size == spec.block_size, (
"Padded KV pages do not support kernel block splitting."
)
page_grid_end = max(order.index(_DIM_L), order.index(_DIM_B)) + 1
page_grid_shape = tuple(shape[dim] for dim in order[:page_grid_end])
assert prod(page_grid_shape) == num_layers * num_blocks, (
"Page padding requires dimensions outside the page tail to be L, B, "
f"or singleton; got {layout.name} with shape {shape}."
)
strides = [0] * 5
current_stride = 1
for physical_idx, dim in reversed(tuple(enumerate(order))):
if padded_page_size is not None and physical_idx == page_grid_end - 1:
current_stride = max(current_stride, padded_page_size)
assert current_stride % padded_page_size == 0
strides[dim] = fixed_strides[dim] or current_stride
current_stride = strides[dim] * shape[dim]
return tuple(strides)
def create_kv_cache_views(
raw: torch.Tensor,
spec: KVCacheSpec,
num_blocks: int,
layout: KVCacheLayout,
kv_cache_tensor: KVCacheTensor,
kernel_block_size: int | None = None,
) -> list[torch.Tensor]:
"""View a flat int8 buffer as one 4D ``[B, H, N, C]`` view per layer.
Block ``b`` of layer ``l`` starts at the tensor offset plus its layer and
block stride contributions.
"""
num_layers = len(kv_cache_tensor.layers)
layer_stride = kv_cache_tensor.layer_stride
block_stride = kv_cache_tensor.block_stride
shape_bytes = compute_layer_kv_cache_shape_bytes(
spec, num_blocks, kernel_block_size
)
ratio = shape_bytes[0] // num_blocks
if ratio > 1:
# Kernel blocks subdivide a manager block into `ratio` equal pieces, so
# they sit a constant stride apart only if a block is one dense page: no
# padding at its end, and no other layer's page before the next block.
dense_page_size = prod(compute_layer_kv_cache_shape_bytes(spec, 1)[1:])
if block_stride != dense_page_size:
raise ValueError(
f"The resolved KV cache layout ({layout.name}) does not store "
"blocks as dense, unpadded pages (block stride "
f"{block_stride} != page {dense_page_size}), so a manager "
f"block cannot be split into {ratio} kernel blocks of "
f"{kernel_block_size} tokens. Reduce --block-size to "
f"{kernel_block_size} or set VLLM_KV_CACHE_LAYOUT to a "
"layer-compact layout (e.g. LBNHC)."
)
assert block_stride % ratio == 0, (
f"Block stride {block_stride} must divide into {ratio} equal kernel blocks."
)
block_stride //= ratio
logical_shape = (num_layers, *shape_bytes)
strides = compute_layout_strides(
spec,
num_blocks,
num_layers,
layout,
kernel_block_size,
fixed_strides=(layer_stride, block_stride, None, None, None),
)
dtype = getattr(spec, "dtype", None)
view_5d = torch.as_strided(
raw,
size=logical_shape,
stride=strides,
storage_offset=raw.storage_offset() + kv_cache_tensor.offset,
)
views = []
for layer_idx in range(num_layers):
cache_logical = view_5d[layer_idx]
if dtype is not None:
cache_logical = cache_logical.view(dtype)
views.append(cache_logical)
return views
@dataclass(frozen=True, kw_only=True)
class AttentionSpec(KVCacheSpec):
num_kv_heads: int
@@ -222,13 +378,16 @@ class AttentionSpec(KVCacheSpec):
head_size_v: int = None # type: ignore[assignment]
kv_quant_mode: KVQuantMode = KVQuantMode.NONE
page_size_padded: int | None = None
indexes_kv_by_block_stride: bool = False
num_head_slots: int | None = None
"""H of the logical ``[B, H, N, C]`` page when packing diverges from one
slot per KV head. None means one slot per KV head. Published by the backend.
"""
state_content_bytes: int | None = None
"""C in bytes when packed; None means dense K/V content."""
tokens_per_state: int | Fraction = 1
"""Tokens covered by one stored state. Ints > 1 compress multiple tokens
into one state (DSv4 sparse MLA); fractions < 1 store multiple states per
token (Whisper block pooling: ``Fraction(1, block_pool_size)``)."""
def __post_init__(self):
if self.head_size_v is None:
@@ -249,7 +408,7 @@ class AttentionSpec(KVCacheSpec):
@property
def unpadded_page_size_bytes(self) -> int:
return self.num_heads * self.storage_block_size * self.state_content_size_bytes
return self.num_heads * self.num_states * self.state_content_size_bytes
@property
def page_size_bytes(self) -> int:
@@ -345,9 +504,9 @@ class FullAttentionSpec(AttentionSpec):
dtype=specs[0].dtype,
kv_quant_mode=specs[0].kv_quant_mode,
page_size_padded=specs[0].page_size_padded,
indexes_kv_by_block_stride=specs[0].indexes_kv_by_block_stride,
num_head_slots=specs[0].num_head_slots,
state_content_bytes=specs[0].state_content_bytes,
tokens_per_state=specs[0].tokens_per_state,
sliding_window=cls.merge_window_sizes(sliding_window),
attention_chunk_size=cls.merge_window_sizes(attention_chunk_size),
# If any layer in the group is non-causal, treat the group as
@@ -384,7 +543,6 @@ class MLAAttentionSpec(FullAttentionSpec):
cache_dtype_str: str | None = None
# DeepseekV4 only fields. Non-DeepseekV4 MLA models leave these at defaults.
alignment: int | None = None # Default to None for no padding.
compress_ratio: int = 1 # Default to 1 for no compression.
model_version: str | None = None
# Marks draft groups that flatten a non-causal query block into decode rows.
non_causal_multi_token_decode: bool = False
@@ -395,28 +553,21 @@ class MLAAttentionSpec(FullAttentionSpec):
super().__post_init__()
_apply_alignment_padding(self)
@property
def storage_block_size(self) -> int:
return self.block_size // self.compress_ratio
@classmethod
def merge(cls, specs: list[Self]) -> Self:
assert all(isinstance(spec, MLAAttentionSpec) for spec in specs), (
"All attention layers in the same KV cache group must be MLAAttentionSpec."
)
cache_dtype_str_set = set(spec.cache_dtype_str for spec in specs)
compress_ratio_set = set(spec.compress_ratio for spec in specs)
tokens_per_state_set = set(spec.tokens_per_state for spec in specs)
model_version_set = set(spec.model_version for spec in specs)
block_stride_set = set(spec.indexes_kv_by_block_stride for spec in specs)
assert (
len(cache_dtype_str_set) == 1
and len(compress_ratio_set) == 1
and len(tokens_per_state_set) == 1
and len(model_version_set) == 1
and len(block_stride_set) == 1
), (
"All attention layers in the same KV cache group must use the same "
"quantization method, compress ratio, model version, and KV block "
"stride indexing."
"quantization method, tokens per state, and model version."
)
merged_spec = cls(
block_size=specs[0].block_size,
@@ -427,9 +578,8 @@ class MLAAttentionSpec(FullAttentionSpec):
page_size_padded=specs[0].page_size_padded,
num_head_slots=specs[0].num_head_slots,
state_content_bytes=specs[0].state_content_bytes,
indexes_kv_by_block_stride=block_stride_set.pop(),
cache_dtype_str=cache_dtype_str_set.pop(),
compress_ratio=compress_ratio_set.pop(),
tokens_per_state=tokens_per_state_set.pop(),
model_version=model_version_set.pop(),
non_causal_multi_token_decode=any(
spec.non_causal_multi_token_decode for spec in specs
@@ -483,9 +633,9 @@ class RSWASpec(FullAttentionSpec):
dtype=base.dtype,
kv_quant_mode=base.kv_quant_mode,
page_size_padded=base.page_size_padded,
indexes_kv_by_block_stride=base.indexes_kv_by_block_stride,
num_head_slots=base.num_head_slots,
state_content_bytes=base.state_content_bytes,
tokens_per_state=base.tokens_per_state,
sliding_window=base.sliding_window,
attention_chunk_size=base.attention_chunk_size,
non_causal=base.non_causal,
@@ -598,8 +748,8 @@ class SlidingWindowMLASpec(SlidingWindowSpec):
cache_dtype_str: str | None = None
# DeepseekV4-only: see MLAAttentionSpec.model_version.
alignment: int | None = None # Default to None for no padding.
compress_ratio: int = 1
model_version: str | None = None
# MLA stores a single latent vector per state; there is no separate V.
head_size_v: int = 0
@@ -610,10 +760,6 @@ class SlidingWindowMLASpec(SlidingWindowSpec):
super().__post_init__()
_apply_alignment_padding(self)
@property
def storage_block_size(self) -> int:
return self.block_size // self.compress_ratio
@classmethod
def merge(cls, specs: list[Self]) -> Self:
assert all(isinstance(spec, SlidingWindowMLASpec) for spec in specs), (
@@ -621,22 +767,20 @@ class SlidingWindowMLASpec(SlidingWindowSpec):
"SlidingWindowMLASpec."
)
cache_dtype_str_set = set(spec.cache_dtype_str for spec in specs)
compress_ratio_set = set(spec.compress_ratio for spec in specs)
tokens_per_state_set = set(spec.tokens_per_state for spec in specs)
model_version_set = set(spec.model_version for spec in specs)
sliding_window_set = set(spec.sliding_window for spec in specs)
block_stride_set = set(spec.indexes_kv_by_block_stride for spec in specs)
extra_retained_set = set(spec.extra_retained_tokens for spec in specs)
assert (
len(cache_dtype_str_set) == 1
and len(compress_ratio_set) == 1
and len(tokens_per_state_set) == 1
and len(model_version_set) == 1
and len(sliding_window_set) == 1
and len(block_stride_set) == 1
and len(extra_retained_set) == 1
), (
"All attention layers in the same KV cache group must use the same "
"quantization method, compress ratio, model version, sliding "
"window size, KV block stride indexing, and retained token count."
"quantization method, tokens per state, model version, sliding "
"window size, and retained token count."
)
return cls(
block_size=specs[0].block_size,
@@ -646,11 +790,10 @@ class SlidingWindowMLASpec(SlidingWindowSpec):
page_size_padded=specs[0].page_size_padded,
num_head_slots=specs[0].num_head_slots,
state_content_bytes=specs[0].state_content_bytes,
indexes_kv_by_block_stride=block_stride_set.pop(),
sliding_window=sliding_window_set.pop(),
extra_retained_tokens=extra_retained_set.pop(),
cache_dtype_str=cache_dtype_str_set.pop(),
compress_ratio=compress_ratio_set.pop(),
tokens_per_state=tokens_per_state_set.pop(),
model_version=model_version_set.pop(),
)
@@ -672,6 +815,15 @@ class MambaSpec(KVCacheSpec):
mamba_type: MambaAttentionBackendEnum = MambaAttentionBackendEnum.MAMBA2
mamba_cache_mode: str = "none"
num_speculative_blocks: int = 0
num_heads: int = 1
tokens_per_state: int = -1
@property
def state_content_size_bytes(self) -> int:
return sum(
prod(shape) * get_dtype_size(dtype)
for (shape, dtype) in zip(self.shapes, self.dtypes)
)
@property
def page_size_bytes(self) -> int:
@@ -771,7 +923,6 @@ class SinkFullAttentionSpec(FullAttentionSpec):
dtype=specs[0].dtype,
kv_quant_mode=specs[0].kv_quant_mode,
page_size_padded=specs[0].page_size_padded,
indexes_kv_by_block_stride=specs[0].indexes_kv_by_block_stride,
num_head_slots=specs[0].num_head_slots,
state_content_bytes=specs[0].state_content_bytes,
sliding_window=cls.merge_window_sizes(sliding_window),
@@ -857,9 +1008,6 @@ class UniformTypeKVCacheSpecs(KVCacheSpec):
return None
# NOTE: below util functions are only used by DeepseekV4 for now.
def get_page_sizes(self) -> list[int]:
return list(set(spec.page_size_bytes for spec in self.kv_cache_specs.values()))
def get_num_layer_tuples(self) -> int:
return Counter(
spec.page_size_bytes for spec in self.kv_cache_specs.values()
@@ -929,12 +1077,24 @@ def get_kv_cache_spec_sliding_window(kv_cache_spec: KVCacheSpec) -> int | None:
class KVCacheTensor:
"""
A class for specifying how the workers should initialize the KV cache.
Placement of a set of same-shaped layers in the KV cache allocation.
Layer ``layers[l]``'s page for block ``b`` starts at
``offset + l * layer_stride + b * block_stride`` bytes into the backing
allocation of ``size`` bytes. Layer-outermost layouts give each layer a
contiguous region (``layer_stride = page * num_blocks``,
``block_stride = page``); block-outermost layouts make each block a
block of all layers' pages (``layer_stride = page``, ``block_stride`` =
the packed block). Tensors whose address ranges overlap
alias the same bytes: cache groups overlay each other, which is sound
because a block ID is owned by one group at a time.
"""
size: int # size of the KV cache tensor in bytes
shared_by: list[str] # layer names that share the same KV cache tensor
offset: int = 0 # byte offset of this layer within a contiguous block
block_stride: int = 0 # total bytes per block in a packed layout (0 = not packed)
size: int # total size of the backing allocation in bytes
layers: list[str] # layer names in L order
layer_stride: int
block_stride: int
offset: int = 0 # byte offset of layers[0]'s block 0
@dataclass
@@ -972,6 +1132,8 @@ class KVCacheConfig:
"""
prefix_cache_retention_interval: int | None = None
"""Resolved retention policy for local prefix-cache checkpoints."""
kv_cache_layout: str | None = None
"""The KV cache layout resolved by the engine core, adopted by all workers."""
@property
def has_mamba_layers(self) -> bool: