[5/N][KV-Cache Layout Refactor] Backend-published KV packing via customize_spec (#51704)

Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
Lucas Wilkinson
2026-08-14 09:47:55 -04:00
committed by GitHub
co-authored by Claude
parent 63a9a5010a
commit 57bd0ed441
18 changed files with 204 additions and 219 deletions
@@ -623,14 +623,17 @@ class Attention(nn.Module, AttentionLayerBase):
# bytes per block. Otherwise (page_size_padded is None) the smallest
# block is fine — ``unify`` scales it up by an integer ratio.
shared_page = vllm_config.cache_config.skip_page_size_padded
sw_per_token = SlidingWindowSpec(
block_size=1,
num_kv_heads=self.num_kv_heads,
head_size=self.head_size,
head_size_v=self.head_size_v,
dtype=self.kv_cache_torch_dtype,
kv_quant_mode=quant_mode,
sliding_window=self.sliding_window,
# The backend owns its packing
sw_per_token = self.attn_backend.customize_spec(
SlidingWindowSpec(
block_size=1,
num_kv_heads=self.num_kv_heads,
head_size=self.head_size,
head_size_v=self.head_size_v,
dtype=self.kv_cache_torch_dtype,
kv_quant_mode=quant_mode,
sliding_window=self.sliding_window,
)
).real_page_size_bytes
sw_block_size = _largest_kernel_block_within(
self.attn_backend,
@@ -649,24 +652,6 @@ class Attention(nn.Module, AttentionLayerBase):
sliding_window=self.sliding_window,
page_size_padded=shared_page,
)
elif self.kv_cache_dtype.startswith("turboquant_"):
from vllm.model_executor.layers.quantization.turboquant.config import (
TurboQuantConfig,
)
from vllm.v1.kv_cache_interface import TQFullAttentionSpec
tq_config = TurboQuantConfig.from_cache_dtype(
self.kv_cache_dtype, self.head_size
)
return TQFullAttentionSpec(
block_size=block_size,
num_kv_heads=self.num_kv_heads,
head_size=self.head_size,
head_size_v=self.head_size,
dtype=self.kv_cache_torch_dtype,
kv_quant_mode=quant_mode,
tq_slot_size=tq_config.slot_size_aligned,
)
else:
return FullAttentionSpec(
block_size=block_size,
@@ -204,7 +204,7 @@ import itertools
import math
from abc import abstractmethod
from collections.abc import Callable
from dataclasses import dataclass
from dataclasses import dataclass, replace
from enum import Enum
from math import lcm
from typing import ClassVar, Generic, TypeVar, cast
@@ -270,6 +270,7 @@ from vllm.utils.torch_utils import (
_encode_layer_name,
_resolve_layer_name,
direct_register_custom_op,
get_dtype_size,
is_quantized_kv_cache,
kv_cache_dtype_str_to_dtype,
np_to_pinned_tensor,
@@ -298,6 +299,7 @@ from vllm.v1.attention.selector import get_attn_backend
from vllm.v1.kv_cache_interface import (
AttentionSpec,
KVCacheSpec,
KVQuantMode,
MLAAttentionSpec,
SlidingWindowMLASpec,
get_kv_quant_mode,
@@ -1152,6 +1154,9 @@ class MLAAttention(nn.Module, AttentionLayerBase):
dtype=kv_cache_dtype,
cache_dtype_str=self.kv_cache_dtype,
kv_quant_mode=get_kv_quant_mode(self.kv_cache_dtype),
# fp8_ds_mla: 656-byte custom layout (kv_lora_rank=512 +
# qk_rope_head_dim=64, head_size=576). See flashmla_sparse.py.
state_content_bytes=656 if self.kv_cache_dtype == "fp8_ds_mla" else None,
)
if self.sliding_window is not None:
return SlidingWindowMLASpec(
@@ -1370,6 +1375,20 @@ class _DecodeConcatQuantFP8(QuantFP8):
class MLACommonBackend(AttentionBackend):
@classmethod
def customize_spec(cls, spec: "AttentionSpec") -> "AttentionSpec":
"""Per-token-head modes pack an inline fp32 scale pair after the
latent data (single-sided: ``head_size_v == 0`` for MLA)."""
mode = spec.kv_quant_mode
if spec.state_content_bytes is not None or not mode.is_per_token_head:
return spec
head_size = spec.head_size
if mode == KVQuantMode.INT4_PER_TOKEN_HEAD:
head_size //= 2
scale_bytes = get_dtype_size(torch.float32)
content = head_size * get_dtype_size(spec.dtype) + 2 * scale_bytes
return replace(spec, state_content_bytes=content)
@staticmethod
def get_name() -> str:
return "TRITON_MLA"