[5/N][KV-Cache Layout Refactor] Backend-published KV packing via customize_spec (#51704)

Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
Lucas Wilkinson
2026-08-14 09:47:55 -04:00
committed by GitHub
co-authored by Claude
parent 63a9a5010a
commit 57bd0ed441
18 changed files with 204 additions and 219 deletions
+61 -140
View File
@@ -16,7 +16,7 @@ from typing_extensions import Self
from vllm.logger import init_logger
from vllm.utils.math_utils import cdiv, round_up
from vllm.utils.torch_utils import get_dtype_size, nvfp4_kv_cache_full_dim
from vllm.utils.torch_utils import get_dtype_size
from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum
from vllm.v1.kv_cache_spec_registry import KVCacheSpecRegistry
@@ -46,7 +46,11 @@ class KVQuantMode(IntEnum):
FP8_PER_TOKEN_HEAD = 3 # per-token-head dynamic scales for fp8
INT4_PER_TOKEN_HEAD = 4 # packed 2×int4/byte, RHT + asymmetric zp
NVFP4 = 5 # packed fp4 data + fp8 block scales
TURBOQUANT = 6 # Hadamard-rotated Lloyd-Max quant, packed K+V per slot
# Hadamard-rotated Lloyd-Max quant, packed K+V per slot.
TURBOQUANT_K8V4 = 6
TURBOQUANT_4BIT_NC = 7
TURBOQUANT_K3V4_NC = 8
TURBOQUANT_3BIT_NC = 9
@property
def is_per_token_head(self) -> bool:
@@ -64,8 +68,13 @@ class KVQuantMode(IntEnum):
@property
def is_turboquant(self) -> bool:
"""True for turboquant quantization mode."""
return self == KVQuantMode.TURBOQUANT
"""True for any turboquant quantization mode."""
return self in (
KVQuantMode.TURBOQUANT_K8V4,
KVQuantMode.TURBOQUANT_4BIT_NC,
KVQuantMode.TURBOQUANT_K3V4_NC,
KVQuantMode.TURBOQUANT_3BIT_NC,
)
def get_kv_quant_mode(kv_cache_dtype: str) -> KVQuantMode:
@@ -79,7 +88,7 @@ def get_kv_quant_mode(kv_cache_dtype: str) -> KVQuantMode:
if kv_cache_dtype.startswith("nvfp4"):
return KVQuantMode.NVFP4
if isinstance(kv_cache_dtype, str) and kv_cache_dtype.startswith("turboquant_"):
return KVQuantMode.TURBOQUANT
return KVQuantMode[kv_cache_dtype.upper()]
if isinstance(kv_cache_dtype, str) and kv_cache_dtype.startswith("fp8"):
return KVQuantMode.FP8_PER_TENSOR
return KVQuantMode.NONE
@@ -210,21 +219,37 @@ class AttentionSpec(KVCacheSpec):
num_kv_heads: int
head_size: int
dtype: torch.dtype
head_size_v: int = None # type: ignore[assignment]
kv_quant_mode: KVQuantMode = KVQuantMode.NONE
page_size_padded: int | None = None
indexes_kv_by_block_stride: bool = False
num_head_slots: int | None = None
"""H of the logical ``[B, H, N, C]`` page when packing diverges from one
slot per KV head. None means one slot per KV head. Published by the backend.
"""
state_content_bytes: int | None = None
"""C in bytes when packed; None means dense K/V content."""
def __post_init__(self):
if self.head_size_v is None:
object.__setattr__(self, "head_size_v", self.head_size)
@property
def num_heads(self) -> int:
if self.num_head_slots is not None:
return self.num_head_slots
return self.num_kv_heads
@property
def state_content_size_bytes(self) -> int:
"""Bytes per (head slot, stored state) cell of the page."""
if self.state_content_bytes is not None:
return self.state_content_bytes
return (self.head_size + self.head_size_v) * get_dtype_size(self.dtype)
@property
def unpadded_page_size_bytes(self) -> int:
unpadded = self.real_page_size_bytes
# Per-token-head scales are stored in separate tensors managed
# by the attention backend, but the memory is carved from the
# raw KV cache allocation so it must be budgeted here.
if self.kv_quant_mode.is_per_token_head:
unpadded += (
2 * self.block_size * self.num_kv_heads * get_dtype_size(torch.float32)
)
return unpadded
return self.num_heads * self.storage_block_size * self.state_content_size_bytes
@property
def page_size_bytes(self) -> int:
@@ -235,20 +260,10 @@ class AttentionSpec(KVCacheSpec):
@property
def real_page_size_bytes(self) -> int:
if self.kv_quant_mode.is_nvfp4:
# Packed layout: fp4 data + fp8 block scales per head.
head_dim = nvfp4_kv_cache_full_dim(self.head_size)
elif self.kv_quant_mode == KVQuantMode.INT4_PER_TOKEN_HEAD:
head_dim = self.head_size // 2
else:
head_dim = self.head_size
return (
2
* self.block_size
* self.num_kv_heads
* head_dim
* get_dtype_size(self.dtype)
)
"""Alias of ``unpadded_page_size_bytes``
TODO(lucas): follow up with TPU backend to see if we can remove this property.
"""
return self.unpadded_page_size_bytes
def max_num_blocks_per_req(self, vllm_config: VllmConfig, max_len: int) -> int:
parallel_config = vllm_config.parallel_config
@@ -267,8 +282,6 @@ class FullAttentionSpec(AttentionSpec):
In this case, we use FullAttentionSpec and record the sliding window size.
"""
head_size_v: int = None # type: ignore[assignment]
sliding_window: int | None = None
"""
Default to None for not using sliding window attention.
@@ -284,10 +297,6 @@ class FullAttentionSpec(AttentionSpec):
cache layout itself.
"""
def __post_init__(self):
if self.head_size_v is None:
object.__setattr__(self, "head_size_v", self.head_size)
def max_memory_usage_bytes(self, vllm_config: VllmConfig) -> int:
max_model_len = vllm_config.model_config.max_model_len
dcp_world_size = vllm_config.parallel_config.decode_context_parallel_size
@@ -337,6 +346,8 @@ class FullAttentionSpec(AttentionSpec):
kv_quant_mode=specs[0].kv_quant_mode,
page_size_padded=specs[0].page_size_padded,
indexes_kv_by_block_stride=specs[0].indexes_kv_by_block_stride,
num_head_slots=specs[0].num_head_slots,
state_content_bytes=specs[0].state_content_bytes,
sliding_window=cls.merge_window_sizes(sliding_window),
attention_chunk_size=cls.merge_window_sizes(attention_chunk_size),
# If any layer in the group is non-causal, treat the group as
@@ -357,23 +368,6 @@ class FullAttentionSpec(AttentionSpec):
)
return merged_spec
@property
def real_page_size_bytes(self) -> int:
if self.kv_quant_mode.is_nvfp4:
# Packed layout per head: fp4 data + fp8 block scales.
# fp4 data: head_size//2 bytes (2 fp4 values per byte)
# fp8 block scale: head_size//16 bytes (1 scale per 16 elements)
last_dim = nvfp4_kv_cache_full_dim(
self.head_size
) + nvfp4_kv_cache_full_dim(self.head_size_v)
elif self.kv_quant_mode == KVQuantMode.INT4_PER_TOKEN_HEAD:
last_dim = self.head_size // 2 + self.head_size_v // 2
else:
last_dim = self.head_size + self.head_size_v
return (
self.block_size * self.num_kv_heads * last_dim * get_dtype_size(self.dtype)
)
def _apply_alignment_padding(spec: MLAAttentionSpec | SlidingWindowMLASpec):
if spec.alignment is None:
@@ -384,32 +378,6 @@ def _apply_alignment_padding(spec: MLAAttentionSpec | SlidingWindowMLASpec):
object.__setattr__(spec, "page_size_padded", padded_page_size)
@dataclass(frozen=True, kw_only=True)
class TQFullAttentionSpec(FullAttentionSpec):
"""FullAttentionSpec with TQ-aware page size.
Python equivalent of the C++ TQ4FullAttentionSpec. Overrides
real_page_size_bytes to use TQ slot bytes instead of the raw
head_size * dtype formula.
"""
tq_slot_size: int = 0
@property
def real_page_size_bytes(self) -> int:
if self.tq_slot_size > 0:
return self.block_size * self.num_kv_heads * self.tq_slot_size
return super().real_page_size_bytes
@classmethod
def merge(cls, specs: list[Self]) -> Self:
merged = super().merge(specs)
assert all(s.tq_slot_size == specs[0].tq_slot_size for s in specs), (
"All TQ layers in the same KV cache group must use the same tq_slot_size."
)
return replace(merged, tq_slot_size=specs[0].tq_slot_size)
@dataclass(frozen=True, kw_only=True)
class MLAAttentionSpec(FullAttentionSpec):
# TODO(Lucas/Chen): less hacky way to do this
@@ -420,6 +388,8 @@ class MLAAttentionSpec(FullAttentionSpec):
model_version: str | None = None
# Marks draft groups that flatten a non-causal query block into decode rows.
non_causal_multi_token_decode: bool = False
# MLA stores a single latent vector per state; there is no separate V.
head_size_v: int = 0
def __post_init__(self):
super().__post_init__()
@@ -429,27 +399,6 @@ class MLAAttentionSpec(FullAttentionSpec):
def storage_block_size(self) -> int:
return self.block_size // self.compress_ratio
@property
def real_page_size_bytes(self) -> int:
if self.cache_dtype_str == "fp8_ds_mla":
if self.model_version == "deepseek_v4":
# DeepseekV4: 448B NoPE + 128B RoPE + 8B fp8 scale = 584B per token.
# head_size stays semantic (512); bytes are determined here.
return self.storage_block_size * 584
# V3.2 main MLA: 656-byte custom layout (kv_lora_rank=512 +
# qk_rope_head_dim=64, head_size=576). See flashmla_sparse.py.
return self.block_size * 656
if self.kv_quant_mode == KVQuantMode.INT4_PER_TOKEN_HEAD:
head_dim = self.head_size // 2
else:
head_dim = self.head_size
return (
self.storage_block_size
* self.num_kv_heads
* head_dim
* get_dtype_size(self.dtype)
)
@classmethod
def merge(cls, specs: list[Self]) -> Self:
assert all(isinstance(spec, MLAAttentionSpec) for spec in specs), (
@@ -476,6 +425,8 @@ class MLAAttentionSpec(FullAttentionSpec):
dtype=specs[0].dtype,
kv_quant_mode=specs[0].kv_quant_mode,
page_size_padded=specs[0].page_size_padded,
num_head_slots=specs[0].num_head_slots,
state_content_bytes=specs[0].state_content_bytes,
indexes_kv_by_block_stride=block_stride_set.pop(),
cache_dtype_str=cache_dtype_str_set.pop(),
compress_ratio=compress_ratio_set.pop(),
@@ -533,6 +484,8 @@ class RSWASpec(FullAttentionSpec):
kv_quant_mode=base.kv_quant_mode,
page_size_padded=base.page_size_padded,
indexes_kv_by_block_stride=base.indexes_kv_by_block_stride,
num_head_slots=base.num_head_slots,
state_content_bytes=base.state_content_bytes,
sliding_window=base.sliding_window,
attention_chunk_size=base.attention_chunk_size,
non_causal=base.non_causal,
@@ -583,7 +536,6 @@ class ChunkedLocalAttentionSpec(AttentionSpec):
@dataclass(frozen=True, kw_only=True)
class SlidingWindowSpec(AttentionSpec):
sliding_window: int
head_size_v: int = None # type: ignore[assignment]
# The trailing edge of the window is extended by ``extra_retained_tokens``
# so that those extra trailing tokens' blocks are retained (but not
# attended). This is needed for multi-module spec decoding which can
@@ -591,30 +543,6 @@ class SlidingWindowSpec(AttentionSpec):
# of the sequence, and thus needs to delay freeing/caching of blocks.
extra_retained_tokens: int = 0
def __post_init__(self):
if self.head_size_v is None:
object.__setattr__(self, "head_size_v", self.head_size)
@property
def real_page_size_bytes(self) -> int:
# Mirror ``FullAttentionSpec.real_page_size_bytes`` for NVFP4 KV cache.
if self.kv_quant_mode.is_nvfp4:
last_dim = nvfp4_kv_cache_full_dim(
self.head_size
) + nvfp4_kv_cache_full_dim(self.head_size_v)
return (
self.block_size
* self.num_kv_heads
* last_dim
* get_dtype_size(self.dtype)
)
return (
self.block_size
* self.num_kv_heads
* (self.head_size + self.head_size_v)
* get_dtype_size(self.dtype)
)
def max_admission_blocks_per_request(
self, max_in_flight_tokens: int, max_model_len: int
) -> int:
@@ -672,31 +600,20 @@ class SlidingWindowMLASpec(SlidingWindowSpec):
alignment: int | None = None # Default to None for no padding.
compress_ratio: int = 1
model_version: str | None = None
# MLA stores a single latent vector per state; there is no separate V.
head_size_v: int = 0
def __post_init__(self):
assert self.model_version in (None, "deepseek_v4"), (
f"Unsupported model version: {self.model_version}"
)
super().__post_init__()
_apply_alignment_padding(self)
@property
def storage_block_size(self) -> int:
return self.block_size // self.compress_ratio
@property
def real_page_size_bytes(self) -> int:
if self.model_version == "deepseek_v4" and self.cache_dtype_str == "fp8_ds_mla":
# DeepseekV4 FlashMLA: 448B NoPE + 128B RoPE + 8B fp8 scale = 584B
# per token. FlashInfer's contiguous bf16/fp8 cache falls through to
# the element-size formula below.
return self.storage_block_size * 584
assert self.model_version in (None, "deepseek_v4"), (
f"Unsupported model version: {self.model_version}"
)
return (
self.storage_block_size
* self.num_kv_heads
* self.head_size
* get_dtype_size(self.dtype)
)
@classmethod
def merge(cls, specs: list[Self]) -> Self:
assert all(isinstance(spec, SlidingWindowMLASpec) for spec in specs), (
@@ -727,6 +644,8 @@ class SlidingWindowMLASpec(SlidingWindowSpec):
head_size=specs[0].head_size,
dtype=specs[0].dtype,
page_size_padded=specs[0].page_size_padded,
num_head_slots=specs[0].num_head_slots,
state_content_bytes=specs[0].state_content_bytes,
indexes_kv_by_block_stride=block_stride_set.pop(),
sliding_window=sliding_window_set.pop(),
extra_retained_tokens=extra_retained_set.pop(),
@@ -853,6 +772,8 @@ class SinkFullAttentionSpec(FullAttentionSpec):
kv_quant_mode=specs[0].kv_quant_mode,
page_size_padded=specs[0].page_size_padded,
indexes_kv_by_block_stride=specs[0].indexes_kv_by_block_stride,
num_head_slots=specs[0].num_head_slots,
state_content_bytes=specs[0].state_content_bytes,
sliding_window=cls.merge_window_sizes(sliding_window),
attention_chunk_size=cls.merge_window_sizes(attention_chunk_size),
non_causal=any(spec.non_causal for spec in specs),