[5/N][KV-Cache Layout Refactor] Backend-published KV packing via customize_spec (#51704)
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude
parent
63a9a5010a
commit
57bd0ed441
@@ -623,14 +623,17 @@ class Attention(nn.Module, AttentionLayerBase):
|
||||
# bytes per block. Otherwise (page_size_padded is None) the smallest
|
||||
# block is fine — ``unify`` scales it up by an integer ratio.
|
||||
shared_page = vllm_config.cache_config.skip_page_size_padded
|
||||
sw_per_token = SlidingWindowSpec(
|
||||
block_size=1,
|
||||
num_kv_heads=self.num_kv_heads,
|
||||
head_size=self.head_size,
|
||||
head_size_v=self.head_size_v,
|
||||
dtype=self.kv_cache_torch_dtype,
|
||||
kv_quant_mode=quant_mode,
|
||||
sliding_window=self.sliding_window,
|
||||
# The backend owns its packing
|
||||
sw_per_token = self.attn_backend.customize_spec(
|
||||
SlidingWindowSpec(
|
||||
block_size=1,
|
||||
num_kv_heads=self.num_kv_heads,
|
||||
head_size=self.head_size,
|
||||
head_size_v=self.head_size_v,
|
||||
dtype=self.kv_cache_torch_dtype,
|
||||
kv_quant_mode=quant_mode,
|
||||
sliding_window=self.sliding_window,
|
||||
)
|
||||
).real_page_size_bytes
|
||||
sw_block_size = _largest_kernel_block_within(
|
||||
self.attn_backend,
|
||||
@@ -649,24 +652,6 @@ class Attention(nn.Module, AttentionLayerBase):
|
||||
sliding_window=self.sliding_window,
|
||||
page_size_padded=shared_page,
|
||||
)
|
||||
elif self.kv_cache_dtype.startswith("turboquant_"):
|
||||
from vllm.model_executor.layers.quantization.turboquant.config import (
|
||||
TurboQuantConfig,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import TQFullAttentionSpec
|
||||
|
||||
tq_config = TurboQuantConfig.from_cache_dtype(
|
||||
self.kv_cache_dtype, self.head_size
|
||||
)
|
||||
return TQFullAttentionSpec(
|
||||
block_size=block_size,
|
||||
num_kv_heads=self.num_kv_heads,
|
||||
head_size=self.head_size,
|
||||
head_size_v=self.head_size,
|
||||
dtype=self.kv_cache_torch_dtype,
|
||||
kv_quant_mode=quant_mode,
|
||||
tq_slot_size=tq_config.slot_size_aligned,
|
||||
)
|
||||
else:
|
||||
return FullAttentionSpec(
|
||||
block_size=block_size,
|
||||
|
||||
@@ -204,7 +204,7 @@ import itertools
|
||||
import math
|
||||
from abc import abstractmethod
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass
|
||||
from dataclasses import dataclass, replace
|
||||
from enum import Enum
|
||||
from math import lcm
|
||||
from typing import ClassVar, Generic, TypeVar, cast
|
||||
@@ -270,6 +270,7 @@ from vllm.utils.torch_utils import (
|
||||
_encode_layer_name,
|
||||
_resolve_layer_name,
|
||||
direct_register_custom_op,
|
||||
get_dtype_size,
|
||||
is_quantized_kv_cache,
|
||||
kv_cache_dtype_str_to_dtype,
|
||||
np_to_pinned_tensor,
|
||||
@@ -298,6 +299,7 @@ from vllm.v1.attention.selector import get_attn_backend
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
AttentionSpec,
|
||||
KVCacheSpec,
|
||||
KVQuantMode,
|
||||
MLAAttentionSpec,
|
||||
SlidingWindowMLASpec,
|
||||
get_kv_quant_mode,
|
||||
@@ -1152,6 +1154,9 @@ class MLAAttention(nn.Module, AttentionLayerBase):
|
||||
dtype=kv_cache_dtype,
|
||||
cache_dtype_str=self.kv_cache_dtype,
|
||||
kv_quant_mode=get_kv_quant_mode(self.kv_cache_dtype),
|
||||
# fp8_ds_mla: 656-byte custom layout (kv_lora_rank=512 +
|
||||
# qk_rope_head_dim=64, head_size=576). See flashmla_sparse.py.
|
||||
state_content_bytes=656 if self.kv_cache_dtype == "fp8_ds_mla" else None,
|
||||
)
|
||||
if self.sliding_window is not None:
|
||||
return SlidingWindowMLASpec(
|
||||
@@ -1370,6 +1375,20 @@ class _DecodeConcatQuantFP8(QuantFP8):
|
||||
|
||||
|
||||
class MLACommonBackend(AttentionBackend):
|
||||
@classmethod
|
||||
def customize_spec(cls, spec: "AttentionSpec") -> "AttentionSpec":
|
||||
"""Per-token-head modes pack an inline fp32 scale pair after the
|
||||
latent data (single-sided: ``head_size_v == 0`` for MLA)."""
|
||||
mode = spec.kv_quant_mode
|
||||
if spec.state_content_bytes is not None or not mode.is_per_token_head:
|
||||
return spec
|
||||
head_size = spec.head_size
|
||||
if mode == KVQuantMode.INT4_PER_TOKEN_HEAD:
|
||||
head_size //= 2
|
||||
scale_bytes = get_dtype_size(torch.float32)
|
||||
content = head_size * get_dtype_size(spec.dtype) + 2 * scale_bytes
|
||||
return replace(spec, state_content_bytes=content)
|
||||
|
||||
@staticmethod
|
||||
def get_name() -> str:
|
||||
return "TRITON_MLA"
|
||||
|
||||
Reference in New Issue
Block a user