[5/N][KV-Cache Layout Refactor] Backend-published KV packing via customize_spec (#51704)

Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
Lucas Wilkinson
2026-08-14 09:47:55 -04:00
committed by GitHub
co-authored by Claude
parent 63a9a5010a
commit 57bd0ed441
18 changed files with 204 additions and 219 deletions
+15 -6
View File
@@ -281,7 +281,10 @@ class TestTurboQuantKVCacheSpec:
@pytest.mark.parametrize("preset", ALL_PRESETS)
def test_kv_cache_spec_sets_kv_quant_mode(self, preset):
from vllm.model_executor.layers.attention.attention import Attention
from vllm.v1.kv_cache_interface import KVQuantMode, TQFullAttentionSpec
from vllm.v1.attention.backends.turboquant_attn import (
TurboQuantAttentionBackend,
)
from vllm.v1.kv_cache_interface import FullAttentionSpec
layer = SimpleNamespace(
attn_type="decoder",
@@ -294,10 +297,16 @@ class TestTurboQuantKVCacheSpec:
)
vllm_config = SimpleNamespace(cache_config=SimpleNamespace(block_size=32))
# The layer builds an unpacked spec; the worker's spec-collection
# loop applies TQ slot packing via the backend's customize_spec hook.
spec = Attention.get_kv_cache_spec(layer, vllm_config)
assert isinstance(spec, FullAttentionSpec)
assert spec.kv_quant_mode.is_turboquant
assert spec.state_content_bytes is None
assert isinstance(spec, TQFullAttentionSpec)
assert spec.kv_quant_mode == KVQuantMode.TURBOQUANT
spec = TurboQuantAttentionBackend.customize_spec(spec)
expected_slot = TurboQuantConfig.from_cache_dtype(preset, 128).slot_size_aligned
assert spec.state_content_bytes == expected_slot
class TestTurboQuantWorkspaceReservation:
@@ -333,15 +342,15 @@ class TestTurboQuantWorkspaceReservation:
@staticmethod
def _fake_kv_cache_spec():
from vllm.v1.kv_cache_interface import TQFullAttentionSpec
from vllm.v1.kv_cache_interface import FullAttentionSpec
return TQFullAttentionSpec(
return FullAttentionSpec(
block_size=32,
num_kv_heads=4,
head_size=128,
head_size_v=128,
dtype=torch.uint8,
tq_slot_size=102,
state_content_bytes=102,
)
def test_metadata_builder_reserves_decode_and_continuation_prefill_workspace(
+10 -11
View File
@@ -2766,21 +2766,20 @@ def test_unify_kv_cache_page_size_padding_requires_backend_support():
kv_cache_utils.unify_kv_cache_spec_page_size(specs)
def test_unpadded_page_size_without_quant_matches_real_page():
# Without quantization the offload transfer width is just the raw page.
spec = new_kv_cache_spec()
assert spec.unpadded_page_size_bytes == spec.real_page_size_bytes
assert spec.page_size_bytes == spec.unpadded_page_size_bytes
def test_unpadded_page_size_includes_per_token_head_scales():
# Per-token-head quant carries inline fp32 scales that are carved from the
# raw KV allocation, so they must be budgeted into the offload width.
spec = new_kv_cache_spec(
dtype=torch.uint8, kv_quant_mode=KVQuantMode.FP8_PER_TOKEN_HEAD
# raw KV allocation, so they must be budgeted into the offload width. The
# packing is published by the owning backend's customize_spec hook.
from vllm.v1.attention.backends.triton_attn import TritonAttentionBackend
dense = new_kv_cache_spec(dtype=torch.uint8)
spec = TritonAttentionBackend.customize_spec(
new_kv_cache_spec(
dtype=torch.uint8, kv_quant_mode=KVQuantMode.FP8_PER_TOKEN_HEAD
)
)
scales = 2 * spec.block_size * spec.num_kv_heads * 4
assert spec.unpadded_page_size_bytes == spec.real_page_size_bytes + scales
assert spec.unpadded_page_size_bytes == dense.unpadded_page_size_bytes + scales
assert spec.page_size_bytes == spec.unpadded_page_size_bytes
-11
View File
@@ -34,7 +34,6 @@ from vllm.v1.kv_cache_interface import (
SinkFullAttentionSpec,
SlidingWindowMLASpec,
SlidingWindowSpec,
TQFullAttentionSpec,
UniformTypeKVCacheSpecs,
get_kv_cache_spec_kind,
)
@@ -85,7 +84,6 @@ class _TrulyUnregisteredSpec(KVCacheSpec):
spec_manager_map: dict[type[KVCacheSpec], type[SingleTypeKVCacheManager]] = {
FullAttentionSpec: FullAttentionManager,
TQFullAttentionSpec: FullAttentionManager,
MLAAttentionSpec: FullAttentionManager,
HiddenStateCacheSpec: FullAttentionManager,
SlidingWindowSpec: SlidingWindowManager,
@@ -98,7 +96,6 @@ spec_manager_map: dict[type[KVCacheSpec], type[SingleTypeKVCacheManager]] = {
spec_uniform_base_map: dict[type[KVCacheSpec], type[KVCacheSpec]] = {
FullAttentionSpec: FullAttentionSpec,
TQFullAttentionSpec: FullAttentionSpec,
MLAAttentionSpec: FullAttentionSpec,
HiddenStateCacheSpec: FullAttentionSpec,
SlidingWindowSpec: SlidingWindowSpec,
@@ -113,13 +110,6 @@ spec_args_map: dict[type[KVCacheSpec], dict[str, Any]] = {
FullAttentionSpec: dict(
block_size=64, num_kv_heads=8, head_size=128, dtype=torch.bfloat16
),
TQFullAttentionSpec: dict(
block_size=64,
num_kv_heads=8,
head_size=128,
dtype=torch.bfloat16,
tq_slot_size=256,
),
MLAAttentionSpec: dict(
block_size=64, num_kv_heads=1, head_size=128, dtype=torch.bfloat16
),
@@ -262,7 +252,6 @@ class TestKVCacheSpecRegistry:
def test_full_attention_family_specs_are_uniform(self):
specs = [
make_spec(FullAttentionSpec),
make_spec(TQFullAttentionSpec),
make_spec(MLAAttentionSpec),
make_spec(HiddenStateCacheSpec),
make_spec(SinkFullAttentionSpec),
@@ -93,6 +93,7 @@ def test_dsv4_packed_zeroer_geometry():
cache_dtype_str="fp8_ds_mla",
alignment=576,
model_version="deepseek_v4",
state_content_bytes=584, # >576 to allocate room for scales at back of page
)
for _ in layer_names
]
@@ -623,14 +623,17 @@ class Attention(nn.Module, AttentionLayerBase):
# bytes per block. Otherwise (page_size_padded is None) the smallest
# block is fine — ``unify`` scales it up by an integer ratio.
shared_page = vllm_config.cache_config.skip_page_size_padded
sw_per_token = SlidingWindowSpec(
block_size=1,
num_kv_heads=self.num_kv_heads,
head_size=self.head_size,
head_size_v=self.head_size_v,
dtype=self.kv_cache_torch_dtype,
kv_quant_mode=quant_mode,
sliding_window=self.sliding_window,
# The backend owns its packing
sw_per_token = self.attn_backend.customize_spec(
SlidingWindowSpec(
block_size=1,
num_kv_heads=self.num_kv_heads,
head_size=self.head_size,
head_size_v=self.head_size_v,
dtype=self.kv_cache_torch_dtype,
kv_quant_mode=quant_mode,
sliding_window=self.sliding_window,
)
).real_page_size_bytes
sw_block_size = _largest_kernel_block_within(
self.attn_backend,
@@ -649,24 +652,6 @@ class Attention(nn.Module, AttentionLayerBase):
sliding_window=self.sliding_window,
page_size_padded=shared_page,
)
elif self.kv_cache_dtype.startswith("turboquant_"):
from vllm.model_executor.layers.quantization.turboquant.config import (
TurboQuantConfig,
)
from vllm.v1.kv_cache_interface import TQFullAttentionSpec
tq_config = TurboQuantConfig.from_cache_dtype(
self.kv_cache_dtype, self.head_size
)
return TQFullAttentionSpec(
block_size=block_size,
num_kv_heads=self.num_kv_heads,
head_size=self.head_size,
head_size_v=self.head_size,
dtype=self.kv_cache_torch_dtype,
kv_quant_mode=quant_mode,
tq_slot_size=tq_config.slot_size_aligned,
)
else:
return FullAttentionSpec(
block_size=block_size,
@@ -204,7 +204,7 @@ import itertools
import math
from abc import abstractmethod
from collections.abc import Callable
from dataclasses import dataclass
from dataclasses import dataclass, replace
from enum import Enum
from math import lcm
from typing import ClassVar, Generic, TypeVar, cast
@@ -270,6 +270,7 @@ from vllm.utils.torch_utils import (
_encode_layer_name,
_resolve_layer_name,
direct_register_custom_op,
get_dtype_size,
is_quantized_kv_cache,
kv_cache_dtype_str_to_dtype,
np_to_pinned_tensor,
@@ -298,6 +299,7 @@ from vllm.v1.attention.selector import get_attn_backend
from vllm.v1.kv_cache_interface import (
AttentionSpec,
KVCacheSpec,
KVQuantMode,
MLAAttentionSpec,
SlidingWindowMLASpec,
get_kv_quant_mode,
@@ -1152,6 +1154,9 @@ class MLAAttention(nn.Module, AttentionLayerBase):
dtype=kv_cache_dtype,
cache_dtype_str=self.kv_cache_dtype,
kv_quant_mode=get_kv_quant_mode(self.kv_cache_dtype),
# fp8_ds_mla: 656-byte custom layout (kv_lora_rank=512 +
# qk_rope_head_dim=64, head_size=576). See flashmla_sparse.py.
state_content_bytes=656 if self.kv_cache_dtype == "fp8_ds_mla" else None,
)
if self.sliding_window is not None:
return SlidingWindowMLASpec(
@@ -1370,6 +1375,20 @@ class _DecodeConcatQuantFP8(QuantFP8):
class MLACommonBackend(AttentionBackend):
@classmethod
def customize_spec(cls, spec: "AttentionSpec") -> "AttentionSpec":
"""Per-token-head modes pack an inline fp32 scale pair after the
latent data (single-sided: ``head_size_v == 0`` for MLA)."""
mode = spec.kv_quant_mode
if spec.state_content_bytes is not None or not mode.is_per_token_head:
return spec
head_size = spec.head_size
if mode == KVQuantMode.INT4_PER_TOKEN_HEAD:
head_size //= 2
scale_bytes = get_dtype_size(torch.float32)
content = head_size * get_dtype_size(spec.dtype) + 2 * scale_bytes
return replace(spec, state_content_bytes=content)
@staticmethod
def get_name() -> str:
return "TRITON_MLA"
+3
View File
@@ -670,6 +670,9 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
alignment=576 if uses_fp8_ds_mla_layout else 512,
model_version="deepseek_v4",
kv_quant_mode=get_kv_quant_mode(self.kv_cache_dtype),
# DeepseekV4: 448B NoPE + 128B RoPE + 8B fp8 scale = 584B per token;
# head_size stays semantic (512).
state_content_bytes=584 if uses_fp8_ds_mla_layout else None,
)
+2
View File
@@ -400,6 +400,8 @@ class MultiHeadLatentAttention(nn.Module, AttentionLayerBase):
dtype=kv_cache_dtype,
cache_dtype_str=self.kv_cache_dtype,
kv_quant_mode=get_kv_quant_mode(self.kv_cache_dtype),
# fp8_ds_mla: 656-byte custom layout; see flashmla_sparse.py.
state_content_bytes=656 if self.kv_cache_dtype == "fp8_ds_mla" else None,
non_causal_multi_token_decode=self.non_causal_multi_token_decode,
)
+13 -13
View File
@@ -687,13 +687,15 @@ class Platform:
def per_token_page_bytes(dtype: "torch.dtype", cache_dtype: str) -> int:
"""Bytes one token occupies in one layer, for the given dtype."""
return FullAttentionSpec(
spec = FullAttentionSpec(
block_size=1,
num_kv_heads=model_config.get_num_kv_heads(parallel_config),
head_size=model_config.get_head_size(),
dtype=dtype,
kv_quant_mode=get_kv_quant_mode(cache_dtype),
).page_size_bytes
)
# The backend owns its packing
return backend_cls.customize_spec(spec).page_size_bytes
primary_dtype = (
STR_DTYPE_TO_TORCH_DTYPE[cache_config.cache_dtype]
@@ -809,23 +811,18 @@ class Platform:
# when all attention layers are TQ. With mixed skip+TQ the skip
# layers still use the standard layout — take max so mamba
# padding covers the largest actual page.
from vllm.model_executor.layers.quantization.turboquant.config import (
TurboQuantConfig,
from vllm.v1.attention.backends.turboquant_attn import (
TurboQuantAttentionBackend,
)
from vllm.v1.kv_cache_interface import TQFullAttentionSpec
tq_cfg = TurboQuantConfig.from_cache_dtype(
cache_config.cache_dtype, model_config.get_head_size()
)
tq_page = TQFullAttentionSpec(
tq_spec = FullAttentionSpec(
block_size=1,
num_kv_heads=model_config.get_num_kv_heads(parallel_config),
head_size=model_config.get_head_size(),
head_size_v=model_config.get_head_size(),
dtype=kv_cache_dtype,
kv_quant_mode=kv_quant_mode,
tq_slot_size=tq_cfg.slot_size_aligned,
).page_size_bytes
)
tq_page = TurboQuantAttentionBackend.customize_spec(tq_spec).page_size_bytes
if cache_config.kv_cache_dtype_skip_layers:
skip_page = FullAttentionSpec(
block_size=1,
@@ -840,12 +837,15 @@ class Platform:
else:
attn_page_size_1_token = tq_page
else:
attn_page_size_1_token = FullAttentionSpec(
attn_spec = FullAttentionSpec(
block_size=1,
num_kv_heads=model_config.get_num_kv_heads(parallel_config),
head_size=model_config.get_head_size(),
dtype=kv_cache_dtype,
kv_quant_mode=kv_quant_mode,
)
attn_page_size_1_token = backend_cls.customize_spec(
attn_spec
).page_size_bytes
# Compute mamba page size
+15 -1
View File
@@ -24,7 +24,7 @@ if TYPE_CHECKING:
from vllm.model_executor.layers.quantization.utils.quant_utils import QuantKey
from vllm.platforms.interface import DeviceCapability
from vllm.v1.attention.backends.utils import KVCacheLayoutType
from vllm.v1.kv_cache_interface import KVCacheSpec, KVQuantMode
from vllm.v1.kv_cache_interface import AttentionSpec, KVCacheSpec, KVQuantMode
from vllm.v1.kv_cache_interface import get_kv_quant_mode
@@ -197,6 +197,20 @@ class AttentionBackend(ABC):
return True
return False
@classmethod
def customize_spec(cls, spec: "AttentionSpec") -> "AttentionSpec":
"""Adjust the layer's KV cache spec for this backend's kernels. Used when the
kernels want KV packed in a specific way.
NOTE: temporary compatibility API. Today the Attention layer builds the spec
from the model config and the backend only gets to adjust it post-hoc; the end
state is for the backend to build and return the spec directly, at which point
this hook goes away.
(see: https://github.com/vllm-project/vllm/issues/42449)
"""
return spec
@classmethod
def get_preferred_block_size(cls, default_block_size: int) -> int:
supported_sizes = cls.get_supported_kernel_block_sizes()
+17 -1
View File
@@ -2,7 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Attention layer with FlashInfer."""
from dataclasses import dataclass
from dataclasses import dataclass, replace
from enum import Enum
from functools import partial
from typing import ClassVar
@@ -50,6 +50,7 @@ from vllm.utils.math_utils import cdiv
from vllm.utils.torch_utils import (
PIN_MEMORY,
canonicalize_singleton_dim_strides,
get_dtype_size,
is_quantized_kv_cache,
is_strictly_contiguous,
nvfp4_kv_cache_full_dim,
@@ -387,6 +388,21 @@ class BatchDCPPrefillWrapper:
class FlashInferBackend(AttentionBackend):
@classmethod
def customize_spec(cls, spec: "AttentionSpec") -> "AttentionSpec":
"""NVFP4 stores K and V as separate per-head slots of packed fp4 data
plus fp8 block scales."""
if spec.state_content_bytes is not None or not spec.kv_quant_mode.is_nvfp4:
return spec
hs_k = nvfp4_kv_cache_full_dim(spec.head_size)
hs_v = nvfp4_kv_cache_full_dim(spec.head_size_v)
assert hs_k == hs_v, "nvfp4 with asymmetric K/V head sizes not yet supported"
return replace(
spec,
num_head_slots=2 * spec.num_kv_heads,
state_content_bytes=hs_k * get_dtype_size(spec.dtype),
)
supported_dtypes: ClassVar[list[torch.dtype]] = [torch.float16, torch.bfloat16]
supported_kv_cache_dtypes: ClassVar[list[CacheDType]] = [
"auto",
@@ -97,6 +97,8 @@ class DeepseekV4SWACache(torch.nn.Module, AttentionLayerBase):
dtype=self.dtype,
sliding_window=self.window_size,
cache_dtype_str=self.cache_config.cache_dtype,
# DeepseekV4 fp8_ds_mla: 584B per token (448B NoPE + 128B RoPE + 8B scales)
state_content_bytes=584 if uses_fp8_ds_mla_layout else None,
# 576B for FlashMLA packing; 512B for FlashInfer sparse (#44577).
alignment=576 if uses_fp8_ds_mla_layout else 512,
model_version="deepseek_v4",
+16 -2
View File
@@ -2,7 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""High-Performance Triton-only Attention layer."""
from dataclasses import dataclass
from dataclasses import dataclass, replace
from typing import ClassVar
import torch
@@ -19,7 +19,7 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
from vllm.platforms import current_platform
from vllm.platforms.interface import DeviceCapability
from vllm.utils.math_utils import next_power_of_2
from vllm.utils.torch_utils import is_quantized_kv_cache
from vllm.utils.torch_utils import get_dtype_size, is_quantized_kv_cache
from vllm.v1.attention.backend import (
AttentionBackend,
AttentionCGSupport,
@@ -274,6 +274,20 @@ class TritonAttentionMetadataBuilder(AttentionMetadataBuilder[TritonAttentionMet
class TritonAttentionBackend(AttentionBackend):
@classmethod
def customize_spec(cls, spec: "AttentionSpec") -> "AttentionSpec":
"""Per-token-head modes pack inline fp32 scales after each head's
data, so the content is (data + one scale) per K/V side."""
mode = spec.kv_quant_mode
if spec.state_content_bytes is not None or not mode.is_per_token_head:
return spec
hs_k, hs_v = spec.head_size, spec.head_size_v
if mode == KVQuantMode.INT4_PER_TOKEN_HEAD:
hs_k, hs_v = hs_k // 2, hs_v // 2
scale_bytes = get_dtype_size(torch.float32)
content = (hs_k + hs_v) * get_dtype_size(spec.dtype) + 2 * scale_bytes
return replace(spec, state_content_bytes=content)
supported_dtypes: ClassVar[list[torch.dtype]] = [
torch.float16,
torch.bfloat16,
+16 -1
View File
@@ -19,7 +19,7 @@ Per-head per-position slot layout:
import contextlib
import functools
import math
from dataclasses import dataclass
from dataclasses import dataclass, replace
from typing import Any, ClassVar
import torch
@@ -137,6 +137,21 @@ class TurboQuantAttentionBackend(AttentionBackend):
"turboquant_3bit_nc",
]
@classmethod
def customize_spec(cls, spec: AttentionSpec) -> AttentionSpec:
"""TurboQuant packs K+V into one slot per head."""
if spec.state_content_bytes is not None or not spec.kv_quant_mode.is_turboquant:
return spec
from vllm.model_executor.layers.quantization.turboquant.config import (
TurboQuantConfig,
)
# KVQuantMode member names mirror the preset strings.
tq = TurboQuantConfig.from_cache_dtype(
spec.kv_quant_mode.name.lower(), spec.head_size
)
return replace(spec, state_content_bytes=tq.slot_size_aligned)
@staticmethod
def get_name() -> str:
return "TURBOQUANT"
@@ -28,7 +28,6 @@ from vllm.v1.kv_cache_interface import (
SinkFullAttentionSpec,
SlidingWindowMLASpec,
SlidingWindowSpec,
TQFullAttentionSpec,
)
from vllm.v1.kv_cache_spec_registry import KVCacheSpecRegistry
from vllm.v1.request import Request
@@ -1929,11 +1928,6 @@ def register_all_kvcache_specs(vllm_config):
)
# FullAttentionSpec subclasses — grouped with FullAttentionSpec
KVCacheSpecRegistry.register(
TQFullAttentionSpec,
FullAttentionManager,
uniform_type_base_spec=FullAttentionSpec,
)
KVCacheSpecRegistry.register(
MLAAttentionSpec, FullAttentionManager, uniform_type_base_spec=FullAttentionSpec
)
+61 -140
View File
@@ -16,7 +16,7 @@ from typing_extensions import Self
from vllm.logger import init_logger
from vllm.utils.math_utils import cdiv, round_up
from vllm.utils.torch_utils import get_dtype_size, nvfp4_kv_cache_full_dim
from vllm.utils.torch_utils import get_dtype_size
from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum
from vllm.v1.kv_cache_spec_registry import KVCacheSpecRegistry
@@ -46,7 +46,11 @@ class KVQuantMode(IntEnum):
FP8_PER_TOKEN_HEAD = 3 # per-token-head dynamic scales for fp8
INT4_PER_TOKEN_HEAD = 4 # packed 2×int4/byte, RHT + asymmetric zp
NVFP4 = 5 # packed fp4 data + fp8 block scales
TURBOQUANT = 6 # Hadamard-rotated Lloyd-Max quant, packed K+V per slot
# Hadamard-rotated Lloyd-Max quant, packed K+V per slot.
TURBOQUANT_K8V4 = 6
TURBOQUANT_4BIT_NC = 7
TURBOQUANT_K3V4_NC = 8
TURBOQUANT_3BIT_NC = 9
@property
def is_per_token_head(self) -> bool:
@@ -64,8 +68,13 @@ class KVQuantMode(IntEnum):
@property
def is_turboquant(self) -> bool:
"""True for turboquant quantization mode."""
return self == KVQuantMode.TURBOQUANT
"""True for any turboquant quantization mode."""
return self in (
KVQuantMode.TURBOQUANT_K8V4,
KVQuantMode.TURBOQUANT_4BIT_NC,
KVQuantMode.TURBOQUANT_K3V4_NC,
KVQuantMode.TURBOQUANT_3BIT_NC,
)
def get_kv_quant_mode(kv_cache_dtype: str) -> KVQuantMode:
@@ -79,7 +88,7 @@ def get_kv_quant_mode(kv_cache_dtype: str) -> KVQuantMode:
if kv_cache_dtype.startswith("nvfp4"):
return KVQuantMode.NVFP4
if isinstance(kv_cache_dtype, str) and kv_cache_dtype.startswith("turboquant_"):
return KVQuantMode.TURBOQUANT
return KVQuantMode[kv_cache_dtype.upper()]
if isinstance(kv_cache_dtype, str) and kv_cache_dtype.startswith("fp8"):
return KVQuantMode.FP8_PER_TENSOR
return KVQuantMode.NONE
@@ -210,21 +219,37 @@ class AttentionSpec(KVCacheSpec):
num_kv_heads: int
head_size: int
dtype: torch.dtype
head_size_v: int = None # type: ignore[assignment]
kv_quant_mode: KVQuantMode = KVQuantMode.NONE
page_size_padded: int | None = None
indexes_kv_by_block_stride: bool = False
num_head_slots: int | None = None
"""H of the logical ``[B, H, N, C]`` page when packing diverges from one
slot per KV head. None means one slot per KV head. Published by the backend.
"""
state_content_bytes: int | None = None
"""C in bytes when packed; None means dense K/V content."""
def __post_init__(self):
if self.head_size_v is None:
object.__setattr__(self, "head_size_v", self.head_size)
@property
def num_heads(self) -> int:
if self.num_head_slots is not None:
return self.num_head_slots
return self.num_kv_heads
@property
def state_content_size_bytes(self) -> int:
"""Bytes per (head slot, stored state) cell of the page."""
if self.state_content_bytes is not None:
return self.state_content_bytes
return (self.head_size + self.head_size_v) * get_dtype_size(self.dtype)
@property
def unpadded_page_size_bytes(self) -> int:
unpadded = self.real_page_size_bytes
# Per-token-head scales are stored in separate tensors managed
# by the attention backend, but the memory is carved from the
# raw KV cache allocation so it must be budgeted here.
if self.kv_quant_mode.is_per_token_head:
unpadded += (
2 * self.block_size * self.num_kv_heads * get_dtype_size(torch.float32)
)
return unpadded
return self.num_heads * self.storage_block_size * self.state_content_size_bytes
@property
def page_size_bytes(self) -> int:
@@ -235,20 +260,10 @@ class AttentionSpec(KVCacheSpec):
@property
def real_page_size_bytes(self) -> int:
if self.kv_quant_mode.is_nvfp4:
# Packed layout: fp4 data + fp8 block scales per head.
head_dim = nvfp4_kv_cache_full_dim(self.head_size)
elif self.kv_quant_mode == KVQuantMode.INT4_PER_TOKEN_HEAD:
head_dim = self.head_size // 2
else:
head_dim = self.head_size
return (
2
* self.block_size
* self.num_kv_heads
* head_dim
* get_dtype_size(self.dtype)
)
"""Alias of ``unpadded_page_size_bytes``
TODO(lucas): follow up with TPU backend to see if we can remove this property.
"""
return self.unpadded_page_size_bytes
def max_num_blocks_per_req(self, vllm_config: VllmConfig, max_len: int) -> int:
parallel_config = vllm_config.parallel_config
@@ -267,8 +282,6 @@ class FullAttentionSpec(AttentionSpec):
In this case, we use FullAttentionSpec and record the sliding window size.
"""
head_size_v: int = None # type: ignore[assignment]
sliding_window: int | None = None
"""
Default to None for not using sliding window attention.
@@ -284,10 +297,6 @@ class FullAttentionSpec(AttentionSpec):
cache layout itself.
"""
def __post_init__(self):
if self.head_size_v is None:
object.__setattr__(self, "head_size_v", self.head_size)
def max_memory_usage_bytes(self, vllm_config: VllmConfig) -> int:
max_model_len = vllm_config.model_config.max_model_len
dcp_world_size = vllm_config.parallel_config.decode_context_parallel_size
@@ -337,6 +346,8 @@ class FullAttentionSpec(AttentionSpec):
kv_quant_mode=specs[0].kv_quant_mode,
page_size_padded=specs[0].page_size_padded,
indexes_kv_by_block_stride=specs[0].indexes_kv_by_block_stride,
num_head_slots=specs[0].num_head_slots,
state_content_bytes=specs[0].state_content_bytes,
sliding_window=cls.merge_window_sizes(sliding_window),
attention_chunk_size=cls.merge_window_sizes(attention_chunk_size),
# If any layer in the group is non-causal, treat the group as
@@ -357,23 +368,6 @@ class FullAttentionSpec(AttentionSpec):
)
return merged_spec
@property
def real_page_size_bytes(self) -> int:
if self.kv_quant_mode.is_nvfp4:
# Packed layout per head: fp4 data + fp8 block scales.
# fp4 data: head_size//2 bytes (2 fp4 values per byte)
# fp8 block scale: head_size//16 bytes (1 scale per 16 elements)
last_dim = nvfp4_kv_cache_full_dim(
self.head_size
) + nvfp4_kv_cache_full_dim(self.head_size_v)
elif self.kv_quant_mode == KVQuantMode.INT4_PER_TOKEN_HEAD:
last_dim = self.head_size // 2 + self.head_size_v // 2
else:
last_dim = self.head_size + self.head_size_v
return (
self.block_size * self.num_kv_heads * last_dim * get_dtype_size(self.dtype)
)
def _apply_alignment_padding(spec: MLAAttentionSpec | SlidingWindowMLASpec):
if spec.alignment is None:
@@ -384,32 +378,6 @@ def _apply_alignment_padding(spec: MLAAttentionSpec | SlidingWindowMLASpec):
object.__setattr__(spec, "page_size_padded", padded_page_size)
@dataclass(frozen=True, kw_only=True)
class TQFullAttentionSpec(FullAttentionSpec):
"""FullAttentionSpec with TQ-aware page size.
Python equivalent of the C++ TQ4FullAttentionSpec. Overrides
real_page_size_bytes to use TQ slot bytes instead of the raw
head_size * dtype formula.
"""
tq_slot_size: int = 0
@property
def real_page_size_bytes(self) -> int:
if self.tq_slot_size > 0:
return self.block_size * self.num_kv_heads * self.tq_slot_size
return super().real_page_size_bytes
@classmethod
def merge(cls, specs: list[Self]) -> Self:
merged = super().merge(specs)
assert all(s.tq_slot_size == specs[0].tq_slot_size for s in specs), (
"All TQ layers in the same KV cache group must use the same tq_slot_size."
)
return replace(merged, tq_slot_size=specs[0].tq_slot_size)
@dataclass(frozen=True, kw_only=True)
class MLAAttentionSpec(FullAttentionSpec):
# TODO(Lucas/Chen): less hacky way to do this
@@ -420,6 +388,8 @@ class MLAAttentionSpec(FullAttentionSpec):
model_version: str | None = None
# Marks draft groups that flatten a non-causal query block into decode rows.
non_causal_multi_token_decode: bool = False
# MLA stores a single latent vector per state; there is no separate V.
head_size_v: int = 0
def __post_init__(self):
super().__post_init__()
@@ -429,27 +399,6 @@ class MLAAttentionSpec(FullAttentionSpec):
def storage_block_size(self) -> int:
return self.block_size // self.compress_ratio
@property
def real_page_size_bytes(self) -> int:
if self.cache_dtype_str == "fp8_ds_mla":
if self.model_version == "deepseek_v4":
# DeepseekV4: 448B NoPE + 128B RoPE + 8B fp8 scale = 584B per token.
# head_size stays semantic (512); bytes are determined here.
return self.storage_block_size * 584
# V3.2 main MLA: 656-byte custom layout (kv_lora_rank=512 +
# qk_rope_head_dim=64, head_size=576). See flashmla_sparse.py.
return self.block_size * 656
if self.kv_quant_mode == KVQuantMode.INT4_PER_TOKEN_HEAD:
head_dim = self.head_size // 2
else:
head_dim = self.head_size
return (
self.storage_block_size
* self.num_kv_heads
* head_dim
* get_dtype_size(self.dtype)
)
@classmethod
def merge(cls, specs: list[Self]) -> Self:
assert all(isinstance(spec, MLAAttentionSpec) for spec in specs), (
@@ -476,6 +425,8 @@ class MLAAttentionSpec(FullAttentionSpec):
dtype=specs[0].dtype,
kv_quant_mode=specs[0].kv_quant_mode,
page_size_padded=specs[0].page_size_padded,
num_head_slots=specs[0].num_head_slots,
state_content_bytes=specs[0].state_content_bytes,
indexes_kv_by_block_stride=block_stride_set.pop(),
cache_dtype_str=cache_dtype_str_set.pop(),
compress_ratio=compress_ratio_set.pop(),
@@ -533,6 +484,8 @@ class RSWASpec(FullAttentionSpec):
kv_quant_mode=base.kv_quant_mode,
page_size_padded=base.page_size_padded,
indexes_kv_by_block_stride=base.indexes_kv_by_block_stride,
num_head_slots=base.num_head_slots,
state_content_bytes=base.state_content_bytes,
sliding_window=base.sliding_window,
attention_chunk_size=base.attention_chunk_size,
non_causal=base.non_causal,
@@ -583,7 +536,6 @@ class ChunkedLocalAttentionSpec(AttentionSpec):
@dataclass(frozen=True, kw_only=True)
class SlidingWindowSpec(AttentionSpec):
sliding_window: int
head_size_v: int = None # type: ignore[assignment]
# The trailing edge of the window is extended by ``extra_retained_tokens``
# so that those extra trailing tokens' blocks are retained (but not
# attended). This is needed for multi-module spec decoding which can
@@ -591,30 +543,6 @@ class SlidingWindowSpec(AttentionSpec):
# of the sequence, and thus needs to delay freeing/caching of blocks.
extra_retained_tokens: int = 0
def __post_init__(self):
if self.head_size_v is None:
object.__setattr__(self, "head_size_v", self.head_size)
@property
def real_page_size_bytes(self) -> int:
# Mirror ``FullAttentionSpec.real_page_size_bytes`` for NVFP4 KV cache.
if self.kv_quant_mode.is_nvfp4:
last_dim = nvfp4_kv_cache_full_dim(
self.head_size
) + nvfp4_kv_cache_full_dim(self.head_size_v)
return (
self.block_size
* self.num_kv_heads
* last_dim
* get_dtype_size(self.dtype)
)
return (
self.block_size
* self.num_kv_heads
* (self.head_size + self.head_size_v)
* get_dtype_size(self.dtype)
)
def max_admission_blocks_per_request(
self, max_in_flight_tokens: int, max_model_len: int
) -> int:
@@ -672,31 +600,20 @@ class SlidingWindowMLASpec(SlidingWindowSpec):
alignment: int | None = None # Default to None for no padding.
compress_ratio: int = 1
model_version: str | None = None
# MLA stores a single latent vector per state; there is no separate V.
head_size_v: int = 0
def __post_init__(self):
assert self.model_version in (None, "deepseek_v4"), (
f"Unsupported model version: {self.model_version}"
)
super().__post_init__()
_apply_alignment_padding(self)
@property
def storage_block_size(self) -> int:
return self.block_size // self.compress_ratio
@property
def real_page_size_bytes(self) -> int:
if self.model_version == "deepseek_v4" and self.cache_dtype_str == "fp8_ds_mla":
# DeepseekV4 FlashMLA: 448B NoPE + 128B RoPE + 8B fp8 scale = 584B
# per token. FlashInfer's contiguous bf16/fp8 cache falls through to
# the element-size formula below.
return self.storage_block_size * 584
assert self.model_version in (None, "deepseek_v4"), (
f"Unsupported model version: {self.model_version}"
)
return (
self.storage_block_size
* self.num_kv_heads
* self.head_size
* get_dtype_size(self.dtype)
)
@classmethod
def merge(cls, specs: list[Self]) -> Self:
assert all(isinstance(spec, SlidingWindowMLASpec) for spec in specs), (
@@ -727,6 +644,8 @@ class SlidingWindowMLASpec(SlidingWindowSpec):
head_size=specs[0].head_size,
dtype=specs[0].dtype,
page_size_padded=specs[0].page_size_padded,
num_head_slots=specs[0].num_head_slots,
state_content_bytes=specs[0].state_content_bytes,
indexes_kv_by_block_stride=block_stride_set.pop(),
sliding_window=sliding_window_set.pop(),
extra_retained_tokens=extra_retained_set.pop(),
@@ -853,6 +772,8 @@ class SinkFullAttentionSpec(FullAttentionSpec):
kv_quant_mode=specs[0].kv_quant_mode,
page_size_padded=specs[0].page_size_padded,
indexes_kv_by_block_stride=specs[0].indexes_kv_by_block_stride,
num_head_slots=specs[0].num_head_slots,
state_content_bytes=specs[0].state_content_bytes,
sliding_window=cls.merge_window_sizes(sliding_window),
attention_chunk_size=cls.merge_window_sizes(attention_chunk_size),
non_causal=any(spec.non_causal for spec in specs),
+1
View File
@@ -75,6 +75,7 @@ def get_kv_cache_spec(vllm_config: VllmConfig) -> dict[str, KVCacheSpec]:
with set_current_vllm_config(vllm_config):
indexes = backend.indexes_kv_by_block_stride()
spec = replace(spec, indexes_kv_by_block_stride=indexes)
spec = backend.customize_spec(spec)
kv_cache_spec[layer_name] = spec
return kv_cache_spec
+1
View File
@@ -7925,6 +7925,7 @@ class GPUModelRunner(
with set_current_vllm_config(self.vllm_config):
indexes = backend.indexes_kv_by_block_stride()
spec = replace(spec, indexes_kv_by_block_stride=indexes)
spec = backend.customize_spec(spec)
kv_cache_spec[layer_name] = spec
return kv_cache_spec