[5/N][KV-Cache Layout Refactor] Backend-published KV packing via customize_spec (#51704)
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude
parent
63a9a5010a
commit
57bd0ed441
+61
-140
@@ -16,7 +16,7 @@ from typing_extensions import Self
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.utils.math_utils import cdiv, round_up
|
||||
from vllm.utils.torch_utils import get_dtype_size, nvfp4_kv_cache_full_dim
|
||||
from vllm.utils.torch_utils import get_dtype_size
|
||||
from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum
|
||||
from vllm.v1.kv_cache_spec_registry import KVCacheSpecRegistry
|
||||
|
||||
@@ -46,7 +46,11 @@ class KVQuantMode(IntEnum):
|
||||
FP8_PER_TOKEN_HEAD = 3 # per-token-head dynamic scales for fp8
|
||||
INT4_PER_TOKEN_HEAD = 4 # packed 2×int4/byte, RHT + asymmetric zp
|
||||
NVFP4 = 5 # packed fp4 data + fp8 block scales
|
||||
TURBOQUANT = 6 # Hadamard-rotated Lloyd-Max quant, packed K+V per slot
|
||||
# Hadamard-rotated Lloyd-Max quant, packed K+V per slot.
|
||||
TURBOQUANT_K8V4 = 6
|
||||
TURBOQUANT_4BIT_NC = 7
|
||||
TURBOQUANT_K3V4_NC = 8
|
||||
TURBOQUANT_3BIT_NC = 9
|
||||
|
||||
@property
|
||||
def is_per_token_head(self) -> bool:
|
||||
@@ -64,8 +68,13 @@ class KVQuantMode(IntEnum):
|
||||
|
||||
@property
|
||||
def is_turboquant(self) -> bool:
|
||||
"""True for turboquant quantization mode."""
|
||||
return self == KVQuantMode.TURBOQUANT
|
||||
"""True for any turboquant quantization mode."""
|
||||
return self in (
|
||||
KVQuantMode.TURBOQUANT_K8V4,
|
||||
KVQuantMode.TURBOQUANT_4BIT_NC,
|
||||
KVQuantMode.TURBOQUANT_K3V4_NC,
|
||||
KVQuantMode.TURBOQUANT_3BIT_NC,
|
||||
)
|
||||
|
||||
|
||||
def get_kv_quant_mode(kv_cache_dtype: str) -> KVQuantMode:
|
||||
@@ -79,7 +88,7 @@ def get_kv_quant_mode(kv_cache_dtype: str) -> KVQuantMode:
|
||||
if kv_cache_dtype.startswith("nvfp4"):
|
||||
return KVQuantMode.NVFP4
|
||||
if isinstance(kv_cache_dtype, str) and kv_cache_dtype.startswith("turboquant_"):
|
||||
return KVQuantMode.TURBOQUANT
|
||||
return KVQuantMode[kv_cache_dtype.upper()]
|
||||
if isinstance(kv_cache_dtype, str) and kv_cache_dtype.startswith("fp8"):
|
||||
return KVQuantMode.FP8_PER_TENSOR
|
||||
return KVQuantMode.NONE
|
||||
@@ -210,21 +219,37 @@ class AttentionSpec(KVCacheSpec):
|
||||
num_kv_heads: int
|
||||
head_size: int
|
||||
dtype: torch.dtype
|
||||
head_size_v: int = None # type: ignore[assignment]
|
||||
kv_quant_mode: KVQuantMode = KVQuantMode.NONE
|
||||
page_size_padded: int | None = None
|
||||
indexes_kv_by_block_stride: bool = False
|
||||
num_head_slots: int | None = None
|
||||
"""H of the logical ``[B, H, N, C]`` page when packing diverges from one
|
||||
slot per KV head. None means one slot per KV head. Published by the backend.
|
||||
"""
|
||||
state_content_bytes: int | None = None
|
||||
"""C in bytes when packed; None means dense K/V content."""
|
||||
|
||||
def __post_init__(self):
|
||||
if self.head_size_v is None:
|
||||
object.__setattr__(self, "head_size_v", self.head_size)
|
||||
|
||||
@property
|
||||
def num_heads(self) -> int:
|
||||
if self.num_head_slots is not None:
|
||||
return self.num_head_slots
|
||||
return self.num_kv_heads
|
||||
|
||||
@property
|
||||
def state_content_size_bytes(self) -> int:
|
||||
"""Bytes per (head slot, stored state) cell of the page."""
|
||||
if self.state_content_bytes is not None:
|
||||
return self.state_content_bytes
|
||||
return (self.head_size + self.head_size_v) * get_dtype_size(self.dtype)
|
||||
|
||||
@property
|
||||
def unpadded_page_size_bytes(self) -> int:
|
||||
unpadded = self.real_page_size_bytes
|
||||
# Per-token-head scales are stored in separate tensors managed
|
||||
# by the attention backend, but the memory is carved from the
|
||||
# raw KV cache allocation so it must be budgeted here.
|
||||
if self.kv_quant_mode.is_per_token_head:
|
||||
unpadded += (
|
||||
2 * self.block_size * self.num_kv_heads * get_dtype_size(torch.float32)
|
||||
)
|
||||
return unpadded
|
||||
return self.num_heads * self.storage_block_size * self.state_content_size_bytes
|
||||
|
||||
@property
|
||||
def page_size_bytes(self) -> int:
|
||||
@@ -235,20 +260,10 @@ class AttentionSpec(KVCacheSpec):
|
||||
|
||||
@property
|
||||
def real_page_size_bytes(self) -> int:
|
||||
if self.kv_quant_mode.is_nvfp4:
|
||||
# Packed layout: fp4 data + fp8 block scales per head.
|
||||
head_dim = nvfp4_kv_cache_full_dim(self.head_size)
|
||||
elif self.kv_quant_mode == KVQuantMode.INT4_PER_TOKEN_HEAD:
|
||||
head_dim = self.head_size // 2
|
||||
else:
|
||||
head_dim = self.head_size
|
||||
return (
|
||||
2
|
||||
* self.block_size
|
||||
* self.num_kv_heads
|
||||
* head_dim
|
||||
* get_dtype_size(self.dtype)
|
||||
)
|
||||
"""Alias of ``unpadded_page_size_bytes``
|
||||
TODO(lucas): follow up with TPU backend to see if we can remove this property.
|
||||
"""
|
||||
return self.unpadded_page_size_bytes
|
||||
|
||||
def max_num_blocks_per_req(self, vllm_config: VllmConfig, max_len: int) -> int:
|
||||
parallel_config = vllm_config.parallel_config
|
||||
@@ -267,8 +282,6 @@ class FullAttentionSpec(AttentionSpec):
|
||||
In this case, we use FullAttentionSpec and record the sliding window size.
|
||||
"""
|
||||
|
||||
head_size_v: int = None # type: ignore[assignment]
|
||||
|
||||
sliding_window: int | None = None
|
||||
"""
|
||||
Default to None for not using sliding window attention.
|
||||
@@ -284,10 +297,6 @@ class FullAttentionSpec(AttentionSpec):
|
||||
cache layout itself.
|
||||
"""
|
||||
|
||||
def __post_init__(self):
|
||||
if self.head_size_v is None:
|
||||
object.__setattr__(self, "head_size_v", self.head_size)
|
||||
|
||||
def max_memory_usage_bytes(self, vllm_config: VllmConfig) -> int:
|
||||
max_model_len = vllm_config.model_config.max_model_len
|
||||
dcp_world_size = vllm_config.parallel_config.decode_context_parallel_size
|
||||
@@ -337,6 +346,8 @@ class FullAttentionSpec(AttentionSpec):
|
||||
kv_quant_mode=specs[0].kv_quant_mode,
|
||||
page_size_padded=specs[0].page_size_padded,
|
||||
indexes_kv_by_block_stride=specs[0].indexes_kv_by_block_stride,
|
||||
num_head_slots=specs[0].num_head_slots,
|
||||
state_content_bytes=specs[0].state_content_bytes,
|
||||
sliding_window=cls.merge_window_sizes(sliding_window),
|
||||
attention_chunk_size=cls.merge_window_sizes(attention_chunk_size),
|
||||
# If any layer in the group is non-causal, treat the group as
|
||||
@@ -357,23 +368,6 @@ class FullAttentionSpec(AttentionSpec):
|
||||
)
|
||||
return merged_spec
|
||||
|
||||
@property
|
||||
def real_page_size_bytes(self) -> int:
|
||||
if self.kv_quant_mode.is_nvfp4:
|
||||
# Packed layout per head: fp4 data + fp8 block scales.
|
||||
# fp4 data: head_size//2 bytes (2 fp4 values per byte)
|
||||
# fp8 block scale: head_size//16 bytes (1 scale per 16 elements)
|
||||
last_dim = nvfp4_kv_cache_full_dim(
|
||||
self.head_size
|
||||
) + nvfp4_kv_cache_full_dim(self.head_size_v)
|
||||
elif self.kv_quant_mode == KVQuantMode.INT4_PER_TOKEN_HEAD:
|
||||
last_dim = self.head_size // 2 + self.head_size_v // 2
|
||||
else:
|
||||
last_dim = self.head_size + self.head_size_v
|
||||
return (
|
||||
self.block_size * self.num_kv_heads * last_dim * get_dtype_size(self.dtype)
|
||||
)
|
||||
|
||||
|
||||
def _apply_alignment_padding(spec: MLAAttentionSpec | SlidingWindowMLASpec):
|
||||
if spec.alignment is None:
|
||||
@@ -384,32 +378,6 @@ def _apply_alignment_padding(spec: MLAAttentionSpec | SlidingWindowMLASpec):
|
||||
object.__setattr__(spec, "page_size_padded", padded_page_size)
|
||||
|
||||
|
||||
@dataclass(frozen=True, kw_only=True)
|
||||
class TQFullAttentionSpec(FullAttentionSpec):
|
||||
"""FullAttentionSpec with TQ-aware page size.
|
||||
|
||||
Python equivalent of the C++ TQ4FullAttentionSpec. Overrides
|
||||
real_page_size_bytes to use TQ slot bytes instead of the raw
|
||||
head_size * dtype formula.
|
||||
"""
|
||||
|
||||
tq_slot_size: int = 0
|
||||
|
||||
@property
|
||||
def real_page_size_bytes(self) -> int:
|
||||
if self.tq_slot_size > 0:
|
||||
return self.block_size * self.num_kv_heads * self.tq_slot_size
|
||||
return super().real_page_size_bytes
|
||||
|
||||
@classmethod
|
||||
def merge(cls, specs: list[Self]) -> Self:
|
||||
merged = super().merge(specs)
|
||||
assert all(s.tq_slot_size == specs[0].tq_slot_size for s in specs), (
|
||||
"All TQ layers in the same KV cache group must use the same tq_slot_size."
|
||||
)
|
||||
return replace(merged, tq_slot_size=specs[0].tq_slot_size)
|
||||
|
||||
|
||||
@dataclass(frozen=True, kw_only=True)
|
||||
class MLAAttentionSpec(FullAttentionSpec):
|
||||
# TODO(Lucas/Chen): less hacky way to do this
|
||||
@@ -420,6 +388,8 @@ class MLAAttentionSpec(FullAttentionSpec):
|
||||
model_version: str | None = None
|
||||
# Marks draft groups that flatten a non-causal query block into decode rows.
|
||||
non_causal_multi_token_decode: bool = False
|
||||
# MLA stores a single latent vector per state; there is no separate V.
|
||||
head_size_v: int = 0
|
||||
|
||||
def __post_init__(self):
|
||||
super().__post_init__()
|
||||
@@ -429,27 +399,6 @@ class MLAAttentionSpec(FullAttentionSpec):
|
||||
def storage_block_size(self) -> int:
|
||||
return self.block_size // self.compress_ratio
|
||||
|
||||
@property
|
||||
def real_page_size_bytes(self) -> int:
|
||||
if self.cache_dtype_str == "fp8_ds_mla":
|
||||
if self.model_version == "deepseek_v4":
|
||||
# DeepseekV4: 448B NoPE + 128B RoPE + 8B fp8 scale = 584B per token.
|
||||
# head_size stays semantic (512); bytes are determined here.
|
||||
return self.storage_block_size * 584
|
||||
# V3.2 main MLA: 656-byte custom layout (kv_lora_rank=512 +
|
||||
# qk_rope_head_dim=64, head_size=576). See flashmla_sparse.py.
|
||||
return self.block_size * 656
|
||||
if self.kv_quant_mode == KVQuantMode.INT4_PER_TOKEN_HEAD:
|
||||
head_dim = self.head_size // 2
|
||||
else:
|
||||
head_dim = self.head_size
|
||||
return (
|
||||
self.storage_block_size
|
||||
* self.num_kv_heads
|
||||
* head_dim
|
||||
* get_dtype_size(self.dtype)
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def merge(cls, specs: list[Self]) -> Self:
|
||||
assert all(isinstance(spec, MLAAttentionSpec) for spec in specs), (
|
||||
@@ -476,6 +425,8 @@ class MLAAttentionSpec(FullAttentionSpec):
|
||||
dtype=specs[0].dtype,
|
||||
kv_quant_mode=specs[0].kv_quant_mode,
|
||||
page_size_padded=specs[0].page_size_padded,
|
||||
num_head_slots=specs[0].num_head_slots,
|
||||
state_content_bytes=specs[0].state_content_bytes,
|
||||
indexes_kv_by_block_stride=block_stride_set.pop(),
|
||||
cache_dtype_str=cache_dtype_str_set.pop(),
|
||||
compress_ratio=compress_ratio_set.pop(),
|
||||
@@ -533,6 +484,8 @@ class RSWASpec(FullAttentionSpec):
|
||||
kv_quant_mode=base.kv_quant_mode,
|
||||
page_size_padded=base.page_size_padded,
|
||||
indexes_kv_by_block_stride=base.indexes_kv_by_block_stride,
|
||||
num_head_slots=base.num_head_slots,
|
||||
state_content_bytes=base.state_content_bytes,
|
||||
sliding_window=base.sliding_window,
|
||||
attention_chunk_size=base.attention_chunk_size,
|
||||
non_causal=base.non_causal,
|
||||
@@ -583,7 +536,6 @@ class ChunkedLocalAttentionSpec(AttentionSpec):
|
||||
@dataclass(frozen=True, kw_only=True)
|
||||
class SlidingWindowSpec(AttentionSpec):
|
||||
sliding_window: int
|
||||
head_size_v: int = None # type: ignore[assignment]
|
||||
# The trailing edge of the window is extended by ``extra_retained_tokens``
|
||||
# so that those extra trailing tokens' blocks are retained (but not
|
||||
# attended). This is needed for multi-module spec decoding which can
|
||||
@@ -591,30 +543,6 @@ class SlidingWindowSpec(AttentionSpec):
|
||||
# of the sequence, and thus needs to delay freeing/caching of blocks.
|
||||
extra_retained_tokens: int = 0
|
||||
|
||||
def __post_init__(self):
|
||||
if self.head_size_v is None:
|
||||
object.__setattr__(self, "head_size_v", self.head_size)
|
||||
|
||||
@property
|
||||
def real_page_size_bytes(self) -> int:
|
||||
# Mirror ``FullAttentionSpec.real_page_size_bytes`` for NVFP4 KV cache.
|
||||
if self.kv_quant_mode.is_nvfp4:
|
||||
last_dim = nvfp4_kv_cache_full_dim(
|
||||
self.head_size
|
||||
) + nvfp4_kv_cache_full_dim(self.head_size_v)
|
||||
return (
|
||||
self.block_size
|
||||
* self.num_kv_heads
|
||||
* last_dim
|
||||
* get_dtype_size(self.dtype)
|
||||
)
|
||||
return (
|
||||
self.block_size
|
||||
* self.num_kv_heads
|
||||
* (self.head_size + self.head_size_v)
|
||||
* get_dtype_size(self.dtype)
|
||||
)
|
||||
|
||||
def max_admission_blocks_per_request(
|
||||
self, max_in_flight_tokens: int, max_model_len: int
|
||||
) -> int:
|
||||
@@ -672,31 +600,20 @@ class SlidingWindowMLASpec(SlidingWindowSpec):
|
||||
alignment: int | None = None # Default to None for no padding.
|
||||
compress_ratio: int = 1
|
||||
model_version: str | None = None
|
||||
# MLA stores a single latent vector per state; there is no separate V.
|
||||
head_size_v: int = 0
|
||||
|
||||
def __post_init__(self):
|
||||
assert self.model_version in (None, "deepseek_v4"), (
|
||||
f"Unsupported model version: {self.model_version}"
|
||||
)
|
||||
super().__post_init__()
|
||||
_apply_alignment_padding(self)
|
||||
|
||||
@property
|
||||
def storage_block_size(self) -> int:
|
||||
return self.block_size // self.compress_ratio
|
||||
|
||||
@property
|
||||
def real_page_size_bytes(self) -> int:
|
||||
if self.model_version == "deepseek_v4" and self.cache_dtype_str == "fp8_ds_mla":
|
||||
# DeepseekV4 FlashMLA: 448B NoPE + 128B RoPE + 8B fp8 scale = 584B
|
||||
# per token. FlashInfer's contiguous bf16/fp8 cache falls through to
|
||||
# the element-size formula below.
|
||||
return self.storage_block_size * 584
|
||||
assert self.model_version in (None, "deepseek_v4"), (
|
||||
f"Unsupported model version: {self.model_version}"
|
||||
)
|
||||
return (
|
||||
self.storage_block_size
|
||||
* self.num_kv_heads
|
||||
* self.head_size
|
||||
* get_dtype_size(self.dtype)
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def merge(cls, specs: list[Self]) -> Self:
|
||||
assert all(isinstance(spec, SlidingWindowMLASpec) for spec in specs), (
|
||||
@@ -727,6 +644,8 @@ class SlidingWindowMLASpec(SlidingWindowSpec):
|
||||
head_size=specs[0].head_size,
|
||||
dtype=specs[0].dtype,
|
||||
page_size_padded=specs[0].page_size_padded,
|
||||
num_head_slots=specs[0].num_head_slots,
|
||||
state_content_bytes=specs[0].state_content_bytes,
|
||||
indexes_kv_by_block_stride=block_stride_set.pop(),
|
||||
sliding_window=sliding_window_set.pop(),
|
||||
extra_retained_tokens=extra_retained_set.pop(),
|
||||
@@ -853,6 +772,8 @@ class SinkFullAttentionSpec(FullAttentionSpec):
|
||||
kv_quant_mode=specs[0].kv_quant_mode,
|
||||
page_size_padded=specs[0].page_size_padded,
|
||||
indexes_kv_by_block_stride=specs[0].indexes_kv_by_block_stride,
|
||||
num_head_slots=specs[0].num_head_slots,
|
||||
state_content_bytes=specs[0].state_content_bytes,
|
||||
sliding_window=cls.merge_window_sizes(sliding_window),
|
||||
attention_chunk_size=cls.merge_window_sizes(attention_chunk_size),
|
||||
non_causal=any(spec.non_causal for spec in specs),
|
||||
|
||||
Reference in New Issue
Block a user