[5/N][KV-Cache Layout Refactor] Backend-published KV packing via customize_spec (#51704)
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude
parent
63a9a5010a
commit
57bd0ed441
@@ -281,7 +281,10 @@ class TestTurboQuantKVCacheSpec:
|
||||
@pytest.mark.parametrize("preset", ALL_PRESETS)
|
||||
def test_kv_cache_spec_sets_kv_quant_mode(self, preset):
|
||||
from vllm.model_executor.layers.attention.attention import Attention
|
||||
from vllm.v1.kv_cache_interface import KVQuantMode, TQFullAttentionSpec
|
||||
from vllm.v1.attention.backends.turboquant_attn import (
|
||||
TurboQuantAttentionBackend,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import FullAttentionSpec
|
||||
|
||||
layer = SimpleNamespace(
|
||||
attn_type="decoder",
|
||||
@@ -294,10 +297,16 @@ class TestTurboQuantKVCacheSpec:
|
||||
)
|
||||
vllm_config = SimpleNamespace(cache_config=SimpleNamespace(block_size=32))
|
||||
|
||||
# The layer builds an unpacked spec; the worker's spec-collection
|
||||
# loop applies TQ slot packing via the backend's customize_spec hook.
|
||||
spec = Attention.get_kv_cache_spec(layer, vllm_config)
|
||||
assert isinstance(spec, FullAttentionSpec)
|
||||
assert spec.kv_quant_mode.is_turboquant
|
||||
assert spec.state_content_bytes is None
|
||||
|
||||
assert isinstance(spec, TQFullAttentionSpec)
|
||||
assert spec.kv_quant_mode == KVQuantMode.TURBOQUANT
|
||||
spec = TurboQuantAttentionBackend.customize_spec(spec)
|
||||
expected_slot = TurboQuantConfig.from_cache_dtype(preset, 128).slot_size_aligned
|
||||
assert spec.state_content_bytes == expected_slot
|
||||
|
||||
|
||||
class TestTurboQuantWorkspaceReservation:
|
||||
@@ -333,15 +342,15 @@ class TestTurboQuantWorkspaceReservation:
|
||||
|
||||
@staticmethod
|
||||
def _fake_kv_cache_spec():
|
||||
from vllm.v1.kv_cache_interface import TQFullAttentionSpec
|
||||
from vllm.v1.kv_cache_interface import FullAttentionSpec
|
||||
|
||||
return TQFullAttentionSpec(
|
||||
return FullAttentionSpec(
|
||||
block_size=32,
|
||||
num_kv_heads=4,
|
||||
head_size=128,
|
||||
head_size_v=128,
|
||||
dtype=torch.uint8,
|
||||
tq_slot_size=102,
|
||||
state_content_bytes=102,
|
||||
)
|
||||
|
||||
def test_metadata_builder_reserves_decode_and_continuation_prefill_workspace(
|
||||
|
||||
@@ -2766,21 +2766,20 @@ def test_unify_kv_cache_page_size_padding_requires_backend_support():
|
||||
kv_cache_utils.unify_kv_cache_spec_page_size(specs)
|
||||
|
||||
|
||||
def test_unpadded_page_size_without_quant_matches_real_page():
|
||||
# Without quantization the offload transfer width is just the raw page.
|
||||
spec = new_kv_cache_spec()
|
||||
assert spec.unpadded_page_size_bytes == spec.real_page_size_bytes
|
||||
assert spec.page_size_bytes == spec.unpadded_page_size_bytes
|
||||
|
||||
|
||||
def test_unpadded_page_size_includes_per_token_head_scales():
|
||||
# Per-token-head quant carries inline fp32 scales that are carved from the
|
||||
# raw KV allocation, so they must be budgeted into the offload width.
|
||||
spec = new_kv_cache_spec(
|
||||
dtype=torch.uint8, kv_quant_mode=KVQuantMode.FP8_PER_TOKEN_HEAD
|
||||
# raw KV allocation, so they must be budgeted into the offload width. The
|
||||
# packing is published by the owning backend's customize_spec hook.
|
||||
from vllm.v1.attention.backends.triton_attn import TritonAttentionBackend
|
||||
|
||||
dense = new_kv_cache_spec(dtype=torch.uint8)
|
||||
spec = TritonAttentionBackend.customize_spec(
|
||||
new_kv_cache_spec(
|
||||
dtype=torch.uint8, kv_quant_mode=KVQuantMode.FP8_PER_TOKEN_HEAD
|
||||
)
|
||||
)
|
||||
scales = 2 * spec.block_size * spec.num_kv_heads * 4
|
||||
assert spec.unpadded_page_size_bytes == spec.real_page_size_bytes + scales
|
||||
assert spec.unpadded_page_size_bytes == dense.unpadded_page_size_bytes + scales
|
||||
assert spec.page_size_bytes == spec.unpadded_page_size_bytes
|
||||
|
||||
|
||||
|
||||
@@ -34,7 +34,6 @@ from vllm.v1.kv_cache_interface import (
|
||||
SinkFullAttentionSpec,
|
||||
SlidingWindowMLASpec,
|
||||
SlidingWindowSpec,
|
||||
TQFullAttentionSpec,
|
||||
UniformTypeKVCacheSpecs,
|
||||
get_kv_cache_spec_kind,
|
||||
)
|
||||
@@ -85,7 +84,6 @@ class _TrulyUnregisteredSpec(KVCacheSpec):
|
||||
|
||||
spec_manager_map: dict[type[KVCacheSpec], type[SingleTypeKVCacheManager]] = {
|
||||
FullAttentionSpec: FullAttentionManager,
|
||||
TQFullAttentionSpec: FullAttentionManager,
|
||||
MLAAttentionSpec: FullAttentionManager,
|
||||
HiddenStateCacheSpec: FullAttentionManager,
|
||||
SlidingWindowSpec: SlidingWindowManager,
|
||||
@@ -98,7 +96,6 @@ spec_manager_map: dict[type[KVCacheSpec], type[SingleTypeKVCacheManager]] = {
|
||||
|
||||
spec_uniform_base_map: dict[type[KVCacheSpec], type[KVCacheSpec]] = {
|
||||
FullAttentionSpec: FullAttentionSpec,
|
||||
TQFullAttentionSpec: FullAttentionSpec,
|
||||
MLAAttentionSpec: FullAttentionSpec,
|
||||
HiddenStateCacheSpec: FullAttentionSpec,
|
||||
SlidingWindowSpec: SlidingWindowSpec,
|
||||
@@ -113,13 +110,6 @@ spec_args_map: dict[type[KVCacheSpec], dict[str, Any]] = {
|
||||
FullAttentionSpec: dict(
|
||||
block_size=64, num_kv_heads=8, head_size=128, dtype=torch.bfloat16
|
||||
),
|
||||
TQFullAttentionSpec: dict(
|
||||
block_size=64,
|
||||
num_kv_heads=8,
|
||||
head_size=128,
|
||||
dtype=torch.bfloat16,
|
||||
tq_slot_size=256,
|
||||
),
|
||||
MLAAttentionSpec: dict(
|
||||
block_size=64, num_kv_heads=1, head_size=128, dtype=torch.bfloat16
|
||||
),
|
||||
@@ -262,7 +252,6 @@ class TestKVCacheSpecRegistry:
|
||||
def test_full_attention_family_specs_are_uniform(self):
|
||||
specs = [
|
||||
make_spec(FullAttentionSpec),
|
||||
make_spec(TQFullAttentionSpec),
|
||||
make_spec(MLAAttentionSpec),
|
||||
make_spec(HiddenStateCacheSpec),
|
||||
make_spec(SinkFullAttentionSpec),
|
||||
|
||||
@@ -93,6 +93,7 @@ def test_dsv4_packed_zeroer_geometry():
|
||||
cache_dtype_str="fp8_ds_mla",
|
||||
alignment=576,
|
||||
model_version="deepseek_v4",
|
||||
state_content_bytes=584, # >576 to allocate room for scales at back of page
|
||||
)
|
||||
for _ in layer_names
|
||||
]
|
||||
|
||||
@@ -623,14 +623,17 @@ class Attention(nn.Module, AttentionLayerBase):
|
||||
# bytes per block. Otherwise (page_size_padded is None) the smallest
|
||||
# block is fine — ``unify`` scales it up by an integer ratio.
|
||||
shared_page = vllm_config.cache_config.skip_page_size_padded
|
||||
sw_per_token = SlidingWindowSpec(
|
||||
block_size=1,
|
||||
num_kv_heads=self.num_kv_heads,
|
||||
head_size=self.head_size,
|
||||
head_size_v=self.head_size_v,
|
||||
dtype=self.kv_cache_torch_dtype,
|
||||
kv_quant_mode=quant_mode,
|
||||
sliding_window=self.sliding_window,
|
||||
# The backend owns its packing
|
||||
sw_per_token = self.attn_backend.customize_spec(
|
||||
SlidingWindowSpec(
|
||||
block_size=1,
|
||||
num_kv_heads=self.num_kv_heads,
|
||||
head_size=self.head_size,
|
||||
head_size_v=self.head_size_v,
|
||||
dtype=self.kv_cache_torch_dtype,
|
||||
kv_quant_mode=quant_mode,
|
||||
sliding_window=self.sliding_window,
|
||||
)
|
||||
).real_page_size_bytes
|
||||
sw_block_size = _largest_kernel_block_within(
|
||||
self.attn_backend,
|
||||
@@ -649,24 +652,6 @@ class Attention(nn.Module, AttentionLayerBase):
|
||||
sliding_window=self.sliding_window,
|
||||
page_size_padded=shared_page,
|
||||
)
|
||||
elif self.kv_cache_dtype.startswith("turboquant_"):
|
||||
from vllm.model_executor.layers.quantization.turboquant.config import (
|
||||
TurboQuantConfig,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import TQFullAttentionSpec
|
||||
|
||||
tq_config = TurboQuantConfig.from_cache_dtype(
|
||||
self.kv_cache_dtype, self.head_size
|
||||
)
|
||||
return TQFullAttentionSpec(
|
||||
block_size=block_size,
|
||||
num_kv_heads=self.num_kv_heads,
|
||||
head_size=self.head_size,
|
||||
head_size_v=self.head_size,
|
||||
dtype=self.kv_cache_torch_dtype,
|
||||
kv_quant_mode=quant_mode,
|
||||
tq_slot_size=tq_config.slot_size_aligned,
|
||||
)
|
||||
else:
|
||||
return FullAttentionSpec(
|
||||
block_size=block_size,
|
||||
|
||||
@@ -204,7 +204,7 @@ import itertools
|
||||
import math
|
||||
from abc import abstractmethod
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass
|
||||
from dataclasses import dataclass, replace
|
||||
from enum import Enum
|
||||
from math import lcm
|
||||
from typing import ClassVar, Generic, TypeVar, cast
|
||||
@@ -270,6 +270,7 @@ from vllm.utils.torch_utils import (
|
||||
_encode_layer_name,
|
||||
_resolve_layer_name,
|
||||
direct_register_custom_op,
|
||||
get_dtype_size,
|
||||
is_quantized_kv_cache,
|
||||
kv_cache_dtype_str_to_dtype,
|
||||
np_to_pinned_tensor,
|
||||
@@ -298,6 +299,7 @@ from vllm.v1.attention.selector import get_attn_backend
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
AttentionSpec,
|
||||
KVCacheSpec,
|
||||
KVQuantMode,
|
||||
MLAAttentionSpec,
|
||||
SlidingWindowMLASpec,
|
||||
get_kv_quant_mode,
|
||||
@@ -1152,6 +1154,9 @@ class MLAAttention(nn.Module, AttentionLayerBase):
|
||||
dtype=kv_cache_dtype,
|
||||
cache_dtype_str=self.kv_cache_dtype,
|
||||
kv_quant_mode=get_kv_quant_mode(self.kv_cache_dtype),
|
||||
# fp8_ds_mla: 656-byte custom layout (kv_lora_rank=512 +
|
||||
# qk_rope_head_dim=64, head_size=576). See flashmla_sparse.py.
|
||||
state_content_bytes=656 if self.kv_cache_dtype == "fp8_ds_mla" else None,
|
||||
)
|
||||
if self.sliding_window is not None:
|
||||
return SlidingWindowMLASpec(
|
||||
@@ -1370,6 +1375,20 @@ class _DecodeConcatQuantFP8(QuantFP8):
|
||||
|
||||
|
||||
class MLACommonBackend(AttentionBackend):
|
||||
@classmethod
|
||||
def customize_spec(cls, spec: "AttentionSpec") -> "AttentionSpec":
|
||||
"""Per-token-head modes pack an inline fp32 scale pair after the
|
||||
latent data (single-sided: ``head_size_v == 0`` for MLA)."""
|
||||
mode = spec.kv_quant_mode
|
||||
if spec.state_content_bytes is not None or not mode.is_per_token_head:
|
||||
return spec
|
||||
head_size = spec.head_size
|
||||
if mode == KVQuantMode.INT4_PER_TOKEN_HEAD:
|
||||
head_size //= 2
|
||||
scale_bytes = get_dtype_size(torch.float32)
|
||||
content = head_size * get_dtype_size(spec.dtype) + 2 * scale_bytes
|
||||
return replace(spec, state_content_bytes=content)
|
||||
|
||||
@staticmethod
|
||||
def get_name() -> str:
|
||||
return "TRITON_MLA"
|
||||
|
||||
@@ -670,6 +670,9 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
|
||||
alignment=576 if uses_fp8_ds_mla_layout else 512,
|
||||
model_version="deepseek_v4",
|
||||
kv_quant_mode=get_kv_quant_mode(self.kv_cache_dtype),
|
||||
# DeepseekV4: 448B NoPE + 128B RoPE + 8B fp8 scale = 584B per token;
|
||||
# head_size stays semantic (512).
|
||||
state_content_bytes=584 if uses_fp8_ds_mla_layout else None,
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -400,6 +400,8 @@ class MultiHeadLatentAttention(nn.Module, AttentionLayerBase):
|
||||
dtype=kv_cache_dtype,
|
||||
cache_dtype_str=self.kv_cache_dtype,
|
||||
kv_quant_mode=get_kv_quant_mode(self.kv_cache_dtype),
|
||||
# fp8_ds_mla: 656-byte custom layout; see flashmla_sparse.py.
|
||||
state_content_bytes=656 if self.kv_cache_dtype == "fp8_ds_mla" else None,
|
||||
non_causal_multi_token_decode=self.non_causal_multi_token_decode,
|
||||
)
|
||||
|
||||
|
||||
+13
-13
@@ -687,13 +687,15 @@ class Platform:
|
||||
|
||||
def per_token_page_bytes(dtype: "torch.dtype", cache_dtype: str) -> int:
|
||||
"""Bytes one token occupies in one layer, for the given dtype."""
|
||||
return FullAttentionSpec(
|
||||
spec = FullAttentionSpec(
|
||||
block_size=1,
|
||||
num_kv_heads=model_config.get_num_kv_heads(parallel_config),
|
||||
head_size=model_config.get_head_size(),
|
||||
dtype=dtype,
|
||||
kv_quant_mode=get_kv_quant_mode(cache_dtype),
|
||||
).page_size_bytes
|
||||
)
|
||||
# The backend owns its packing
|
||||
return backend_cls.customize_spec(spec).page_size_bytes
|
||||
|
||||
primary_dtype = (
|
||||
STR_DTYPE_TO_TORCH_DTYPE[cache_config.cache_dtype]
|
||||
@@ -809,23 +811,18 @@ class Platform:
|
||||
# when all attention layers are TQ. With mixed skip+TQ the skip
|
||||
# layers still use the standard layout — take max so mamba
|
||||
# padding covers the largest actual page.
|
||||
from vllm.model_executor.layers.quantization.turboquant.config import (
|
||||
TurboQuantConfig,
|
||||
from vllm.v1.attention.backends.turboquant_attn import (
|
||||
TurboQuantAttentionBackend,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import TQFullAttentionSpec
|
||||
|
||||
tq_cfg = TurboQuantConfig.from_cache_dtype(
|
||||
cache_config.cache_dtype, model_config.get_head_size()
|
||||
)
|
||||
tq_page = TQFullAttentionSpec(
|
||||
tq_spec = FullAttentionSpec(
|
||||
block_size=1,
|
||||
num_kv_heads=model_config.get_num_kv_heads(parallel_config),
|
||||
head_size=model_config.get_head_size(),
|
||||
head_size_v=model_config.get_head_size(),
|
||||
dtype=kv_cache_dtype,
|
||||
kv_quant_mode=kv_quant_mode,
|
||||
tq_slot_size=tq_cfg.slot_size_aligned,
|
||||
).page_size_bytes
|
||||
)
|
||||
tq_page = TurboQuantAttentionBackend.customize_spec(tq_spec).page_size_bytes
|
||||
if cache_config.kv_cache_dtype_skip_layers:
|
||||
skip_page = FullAttentionSpec(
|
||||
block_size=1,
|
||||
@@ -840,12 +837,15 @@ class Platform:
|
||||
else:
|
||||
attn_page_size_1_token = tq_page
|
||||
else:
|
||||
attn_page_size_1_token = FullAttentionSpec(
|
||||
attn_spec = FullAttentionSpec(
|
||||
block_size=1,
|
||||
num_kv_heads=model_config.get_num_kv_heads(parallel_config),
|
||||
head_size=model_config.get_head_size(),
|
||||
dtype=kv_cache_dtype,
|
||||
kv_quant_mode=kv_quant_mode,
|
||||
)
|
||||
attn_page_size_1_token = backend_cls.customize_spec(
|
||||
attn_spec
|
||||
).page_size_bytes
|
||||
|
||||
# Compute mamba page size
|
||||
|
||||
@@ -24,7 +24,7 @@ if TYPE_CHECKING:
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import QuantKey
|
||||
from vllm.platforms.interface import DeviceCapability
|
||||
from vllm.v1.attention.backends.utils import KVCacheLayoutType
|
||||
from vllm.v1.kv_cache_interface import KVCacheSpec, KVQuantMode
|
||||
from vllm.v1.kv_cache_interface import AttentionSpec, KVCacheSpec, KVQuantMode
|
||||
|
||||
from vllm.v1.kv_cache_interface import get_kv_quant_mode
|
||||
|
||||
@@ -197,6 +197,20 @@ class AttentionBackend(ABC):
|
||||
return True
|
||||
return False
|
||||
|
||||
@classmethod
|
||||
def customize_spec(cls, spec: "AttentionSpec") -> "AttentionSpec":
|
||||
"""Adjust the layer's KV cache spec for this backend's kernels. Used when the
|
||||
kernels want KV packed in a specific way.
|
||||
|
||||
NOTE: temporary compatibility API. Today the Attention layer builds the spec
|
||||
from the model config and the backend only gets to adjust it post-hoc; the end
|
||||
state is for the backend to build and return the spec directly, at which point
|
||||
this hook goes away.
|
||||
|
||||
(see: https://github.com/vllm-project/vllm/issues/42449)
|
||||
"""
|
||||
return spec
|
||||
|
||||
@classmethod
|
||||
def get_preferred_block_size(cls, default_block_size: int) -> int:
|
||||
supported_sizes = cls.get_supported_kernel_block_sizes()
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Attention layer with FlashInfer."""
|
||||
|
||||
from dataclasses import dataclass
|
||||
from dataclasses import dataclass, replace
|
||||
from enum import Enum
|
||||
from functools import partial
|
||||
from typing import ClassVar
|
||||
@@ -50,6 +50,7 @@ from vllm.utils.math_utils import cdiv
|
||||
from vllm.utils.torch_utils import (
|
||||
PIN_MEMORY,
|
||||
canonicalize_singleton_dim_strides,
|
||||
get_dtype_size,
|
||||
is_quantized_kv_cache,
|
||||
is_strictly_contiguous,
|
||||
nvfp4_kv_cache_full_dim,
|
||||
@@ -387,6 +388,21 @@ class BatchDCPPrefillWrapper:
|
||||
|
||||
|
||||
class FlashInferBackend(AttentionBackend):
|
||||
@classmethod
|
||||
def customize_spec(cls, spec: "AttentionSpec") -> "AttentionSpec":
|
||||
"""NVFP4 stores K and V as separate per-head slots of packed fp4 data
|
||||
plus fp8 block scales."""
|
||||
if spec.state_content_bytes is not None or not spec.kv_quant_mode.is_nvfp4:
|
||||
return spec
|
||||
hs_k = nvfp4_kv_cache_full_dim(spec.head_size)
|
||||
hs_v = nvfp4_kv_cache_full_dim(spec.head_size_v)
|
||||
assert hs_k == hs_v, "nvfp4 with asymmetric K/V head sizes not yet supported"
|
||||
return replace(
|
||||
spec,
|
||||
num_head_slots=2 * spec.num_kv_heads,
|
||||
state_content_bytes=hs_k * get_dtype_size(spec.dtype),
|
||||
)
|
||||
|
||||
supported_dtypes: ClassVar[list[torch.dtype]] = [torch.float16, torch.bfloat16]
|
||||
supported_kv_cache_dtypes: ClassVar[list[CacheDType]] = [
|
||||
"auto",
|
||||
|
||||
@@ -97,6 +97,8 @@ class DeepseekV4SWACache(torch.nn.Module, AttentionLayerBase):
|
||||
dtype=self.dtype,
|
||||
sliding_window=self.window_size,
|
||||
cache_dtype_str=self.cache_config.cache_dtype,
|
||||
# DeepseekV4 fp8_ds_mla: 584B per token (448B NoPE + 128B RoPE + 8B scales)
|
||||
state_content_bytes=584 if uses_fp8_ds_mla_layout else None,
|
||||
# 576B for FlashMLA packing; 512B for FlashInfer sparse (#44577).
|
||||
alignment=576 if uses_fp8_ds_mla_layout else 512,
|
||||
model_version="deepseek_v4",
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""High-Performance Triton-only Attention layer."""
|
||||
|
||||
from dataclasses import dataclass
|
||||
from dataclasses import dataclass, replace
|
||||
from typing import ClassVar
|
||||
|
||||
import torch
|
||||
@@ -19,7 +19,7 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.platforms.interface import DeviceCapability
|
||||
from vllm.utils.math_utils import next_power_of_2
|
||||
from vllm.utils.torch_utils import is_quantized_kv_cache
|
||||
from vllm.utils.torch_utils import get_dtype_size, is_quantized_kv_cache
|
||||
from vllm.v1.attention.backend import (
|
||||
AttentionBackend,
|
||||
AttentionCGSupport,
|
||||
@@ -274,6 +274,20 @@ class TritonAttentionMetadataBuilder(AttentionMetadataBuilder[TritonAttentionMet
|
||||
|
||||
|
||||
class TritonAttentionBackend(AttentionBackend):
|
||||
@classmethod
|
||||
def customize_spec(cls, spec: "AttentionSpec") -> "AttentionSpec":
|
||||
"""Per-token-head modes pack inline fp32 scales after each head's
|
||||
data, so the content is (data + one scale) per K/V side."""
|
||||
mode = spec.kv_quant_mode
|
||||
if spec.state_content_bytes is not None or not mode.is_per_token_head:
|
||||
return spec
|
||||
hs_k, hs_v = spec.head_size, spec.head_size_v
|
||||
if mode == KVQuantMode.INT4_PER_TOKEN_HEAD:
|
||||
hs_k, hs_v = hs_k // 2, hs_v // 2
|
||||
scale_bytes = get_dtype_size(torch.float32)
|
||||
content = (hs_k + hs_v) * get_dtype_size(spec.dtype) + 2 * scale_bytes
|
||||
return replace(spec, state_content_bytes=content)
|
||||
|
||||
supported_dtypes: ClassVar[list[torch.dtype]] = [
|
||||
torch.float16,
|
||||
torch.bfloat16,
|
||||
|
||||
@@ -19,7 +19,7 @@ Per-head per-position slot layout:
|
||||
import contextlib
|
||||
import functools
|
||||
import math
|
||||
from dataclasses import dataclass
|
||||
from dataclasses import dataclass, replace
|
||||
from typing import Any, ClassVar
|
||||
|
||||
import torch
|
||||
@@ -137,6 +137,21 @@ class TurboQuantAttentionBackend(AttentionBackend):
|
||||
"turboquant_3bit_nc",
|
||||
]
|
||||
|
||||
@classmethod
|
||||
def customize_spec(cls, spec: AttentionSpec) -> AttentionSpec:
|
||||
"""TurboQuant packs K+V into one slot per head."""
|
||||
if spec.state_content_bytes is not None or not spec.kv_quant_mode.is_turboquant:
|
||||
return spec
|
||||
from vllm.model_executor.layers.quantization.turboquant.config import (
|
||||
TurboQuantConfig,
|
||||
)
|
||||
|
||||
# KVQuantMode member names mirror the preset strings.
|
||||
tq = TurboQuantConfig.from_cache_dtype(
|
||||
spec.kv_quant_mode.name.lower(), spec.head_size
|
||||
)
|
||||
return replace(spec, state_content_bytes=tq.slot_size_aligned)
|
||||
|
||||
@staticmethod
|
||||
def get_name() -> str:
|
||||
return "TURBOQUANT"
|
||||
|
||||
@@ -28,7 +28,6 @@ from vllm.v1.kv_cache_interface import (
|
||||
SinkFullAttentionSpec,
|
||||
SlidingWindowMLASpec,
|
||||
SlidingWindowSpec,
|
||||
TQFullAttentionSpec,
|
||||
)
|
||||
from vllm.v1.kv_cache_spec_registry import KVCacheSpecRegistry
|
||||
from vllm.v1.request import Request
|
||||
@@ -1929,11 +1928,6 @@ def register_all_kvcache_specs(vllm_config):
|
||||
)
|
||||
|
||||
# FullAttentionSpec subclasses — grouped with FullAttentionSpec
|
||||
KVCacheSpecRegistry.register(
|
||||
TQFullAttentionSpec,
|
||||
FullAttentionManager,
|
||||
uniform_type_base_spec=FullAttentionSpec,
|
||||
)
|
||||
KVCacheSpecRegistry.register(
|
||||
MLAAttentionSpec, FullAttentionManager, uniform_type_base_spec=FullAttentionSpec
|
||||
)
|
||||
|
||||
+61
-140
@@ -16,7 +16,7 @@ from typing_extensions import Self
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.utils.math_utils import cdiv, round_up
|
||||
from vllm.utils.torch_utils import get_dtype_size, nvfp4_kv_cache_full_dim
|
||||
from vllm.utils.torch_utils import get_dtype_size
|
||||
from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum
|
||||
from vllm.v1.kv_cache_spec_registry import KVCacheSpecRegistry
|
||||
|
||||
@@ -46,7 +46,11 @@ class KVQuantMode(IntEnum):
|
||||
FP8_PER_TOKEN_HEAD = 3 # per-token-head dynamic scales for fp8
|
||||
INT4_PER_TOKEN_HEAD = 4 # packed 2×int4/byte, RHT + asymmetric zp
|
||||
NVFP4 = 5 # packed fp4 data + fp8 block scales
|
||||
TURBOQUANT = 6 # Hadamard-rotated Lloyd-Max quant, packed K+V per slot
|
||||
# Hadamard-rotated Lloyd-Max quant, packed K+V per slot.
|
||||
TURBOQUANT_K8V4 = 6
|
||||
TURBOQUANT_4BIT_NC = 7
|
||||
TURBOQUANT_K3V4_NC = 8
|
||||
TURBOQUANT_3BIT_NC = 9
|
||||
|
||||
@property
|
||||
def is_per_token_head(self) -> bool:
|
||||
@@ -64,8 +68,13 @@ class KVQuantMode(IntEnum):
|
||||
|
||||
@property
|
||||
def is_turboquant(self) -> bool:
|
||||
"""True for turboquant quantization mode."""
|
||||
return self == KVQuantMode.TURBOQUANT
|
||||
"""True for any turboquant quantization mode."""
|
||||
return self in (
|
||||
KVQuantMode.TURBOQUANT_K8V4,
|
||||
KVQuantMode.TURBOQUANT_4BIT_NC,
|
||||
KVQuantMode.TURBOQUANT_K3V4_NC,
|
||||
KVQuantMode.TURBOQUANT_3BIT_NC,
|
||||
)
|
||||
|
||||
|
||||
def get_kv_quant_mode(kv_cache_dtype: str) -> KVQuantMode:
|
||||
@@ -79,7 +88,7 @@ def get_kv_quant_mode(kv_cache_dtype: str) -> KVQuantMode:
|
||||
if kv_cache_dtype.startswith("nvfp4"):
|
||||
return KVQuantMode.NVFP4
|
||||
if isinstance(kv_cache_dtype, str) and kv_cache_dtype.startswith("turboquant_"):
|
||||
return KVQuantMode.TURBOQUANT
|
||||
return KVQuantMode[kv_cache_dtype.upper()]
|
||||
if isinstance(kv_cache_dtype, str) and kv_cache_dtype.startswith("fp8"):
|
||||
return KVQuantMode.FP8_PER_TENSOR
|
||||
return KVQuantMode.NONE
|
||||
@@ -210,21 +219,37 @@ class AttentionSpec(KVCacheSpec):
|
||||
num_kv_heads: int
|
||||
head_size: int
|
||||
dtype: torch.dtype
|
||||
head_size_v: int = None # type: ignore[assignment]
|
||||
kv_quant_mode: KVQuantMode = KVQuantMode.NONE
|
||||
page_size_padded: int | None = None
|
||||
indexes_kv_by_block_stride: bool = False
|
||||
num_head_slots: int | None = None
|
||||
"""H of the logical ``[B, H, N, C]`` page when packing diverges from one
|
||||
slot per KV head. None means one slot per KV head. Published by the backend.
|
||||
"""
|
||||
state_content_bytes: int | None = None
|
||||
"""C in bytes when packed; None means dense K/V content."""
|
||||
|
||||
def __post_init__(self):
|
||||
if self.head_size_v is None:
|
||||
object.__setattr__(self, "head_size_v", self.head_size)
|
||||
|
||||
@property
|
||||
def num_heads(self) -> int:
|
||||
if self.num_head_slots is not None:
|
||||
return self.num_head_slots
|
||||
return self.num_kv_heads
|
||||
|
||||
@property
|
||||
def state_content_size_bytes(self) -> int:
|
||||
"""Bytes per (head slot, stored state) cell of the page."""
|
||||
if self.state_content_bytes is not None:
|
||||
return self.state_content_bytes
|
||||
return (self.head_size + self.head_size_v) * get_dtype_size(self.dtype)
|
||||
|
||||
@property
|
||||
def unpadded_page_size_bytes(self) -> int:
|
||||
unpadded = self.real_page_size_bytes
|
||||
# Per-token-head scales are stored in separate tensors managed
|
||||
# by the attention backend, but the memory is carved from the
|
||||
# raw KV cache allocation so it must be budgeted here.
|
||||
if self.kv_quant_mode.is_per_token_head:
|
||||
unpadded += (
|
||||
2 * self.block_size * self.num_kv_heads * get_dtype_size(torch.float32)
|
||||
)
|
||||
return unpadded
|
||||
return self.num_heads * self.storage_block_size * self.state_content_size_bytes
|
||||
|
||||
@property
|
||||
def page_size_bytes(self) -> int:
|
||||
@@ -235,20 +260,10 @@ class AttentionSpec(KVCacheSpec):
|
||||
|
||||
@property
|
||||
def real_page_size_bytes(self) -> int:
|
||||
if self.kv_quant_mode.is_nvfp4:
|
||||
# Packed layout: fp4 data + fp8 block scales per head.
|
||||
head_dim = nvfp4_kv_cache_full_dim(self.head_size)
|
||||
elif self.kv_quant_mode == KVQuantMode.INT4_PER_TOKEN_HEAD:
|
||||
head_dim = self.head_size // 2
|
||||
else:
|
||||
head_dim = self.head_size
|
||||
return (
|
||||
2
|
||||
* self.block_size
|
||||
* self.num_kv_heads
|
||||
* head_dim
|
||||
* get_dtype_size(self.dtype)
|
||||
)
|
||||
"""Alias of ``unpadded_page_size_bytes``
|
||||
TODO(lucas): follow up with TPU backend to see if we can remove this property.
|
||||
"""
|
||||
return self.unpadded_page_size_bytes
|
||||
|
||||
def max_num_blocks_per_req(self, vllm_config: VllmConfig, max_len: int) -> int:
|
||||
parallel_config = vllm_config.parallel_config
|
||||
@@ -267,8 +282,6 @@ class FullAttentionSpec(AttentionSpec):
|
||||
In this case, we use FullAttentionSpec and record the sliding window size.
|
||||
"""
|
||||
|
||||
head_size_v: int = None # type: ignore[assignment]
|
||||
|
||||
sliding_window: int | None = None
|
||||
"""
|
||||
Default to None for not using sliding window attention.
|
||||
@@ -284,10 +297,6 @@ class FullAttentionSpec(AttentionSpec):
|
||||
cache layout itself.
|
||||
"""
|
||||
|
||||
def __post_init__(self):
|
||||
if self.head_size_v is None:
|
||||
object.__setattr__(self, "head_size_v", self.head_size)
|
||||
|
||||
def max_memory_usage_bytes(self, vllm_config: VllmConfig) -> int:
|
||||
max_model_len = vllm_config.model_config.max_model_len
|
||||
dcp_world_size = vllm_config.parallel_config.decode_context_parallel_size
|
||||
@@ -337,6 +346,8 @@ class FullAttentionSpec(AttentionSpec):
|
||||
kv_quant_mode=specs[0].kv_quant_mode,
|
||||
page_size_padded=specs[0].page_size_padded,
|
||||
indexes_kv_by_block_stride=specs[0].indexes_kv_by_block_stride,
|
||||
num_head_slots=specs[0].num_head_slots,
|
||||
state_content_bytes=specs[0].state_content_bytes,
|
||||
sliding_window=cls.merge_window_sizes(sliding_window),
|
||||
attention_chunk_size=cls.merge_window_sizes(attention_chunk_size),
|
||||
# If any layer in the group is non-causal, treat the group as
|
||||
@@ -357,23 +368,6 @@ class FullAttentionSpec(AttentionSpec):
|
||||
)
|
||||
return merged_spec
|
||||
|
||||
@property
|
||||
def real_page_size_bytes(self) -> int:
|
||||
if self.kv_quant_mode.is_nvfp4:
|
||||
# Packed layout per head: fp4 data + fp8 block scales.
|
||||
# fp4 data: head_size//2 bytes (2 fp4 values per byte)
|
||||
# fp8 block scale: head_size//16 bytes (1 scale per 16 elements)
|
||||
last_dim = nvfp4_kv_cache_full_dim(
|
||||
self.head_size
|
||||
) + nvfp4_kv_cache_full_dim(self.head_size_v)
|
||||
elif self.kv_quant_mode == KVQuantMode.INT4_PER_TOKEN_HEAD:
|
||||
last_dim = self.head_size // 2 + self.head_size_v // 2
|
||||
else:
|
||||
last_dim = self.head_size + self.head_size_v
|
||||
return (
|
||||
self.block_size * self.num_kv_heads * last_dim * get_dtype_size(self.dtype)
|
||||
)
|
||||
|
||||
|
||||
def _apply_alignment_padding(spec: MLAAttentionSpec | SlidingWindowMLASpec):
|
||||
if spec.alignment is None:
|
||||
@@ -384,32 +378,6 @@ def _apply_alignment_padding(spec: MLAAttentionSpec | SlidingWindowMLASpec):
|
||||
object.__setattr__(spec, "page_size_padded", padded_page_size)
|
||||
|
||||
|
||||
@dataclass(frozen=True, kw_only=True)
|
||||
class TQFullAttentionSpec(FullAttentionSpec):
|
||||
"""FullAttentionSpec with TQ-aware page size.
|
||||
|
||||
Python equivalent of the C++ TQ4FullAttentionSpec. Overrides
|
||||
real_page_size_bytes to use TQ slot bytes instead of the raw
|
||||
head_size * dtype formula.
|
||||
"""
|
||||
|
||||
tq_slot_size: int = 0
|
||||
|
||||
@property
|
||||
def real_page_size_bytes(self) -> int:
|
||||
if self.tq_slot_size > 0:
|
||||
return self.block_size * self.num_kv_heads * self.tq_slot_size
|
||||
return super().real_page_size_bytes
|
||||
|
||||
@classmethod
|
||||
def merge(cls, specs: list[Self]) -> Self:
|
||||
merged = super().merge(specs)
|
||||
assert all(s.tq_slot_size == specs[0].tq_slot_size for s in specs), (
|
||||
"All TQ layers in the same KV cache group must use the same tq_slot_size."
|
||||
)
|
||||
return replace(merged, tq_slot_size=specs[0].tq_slot_size)
|
||||
|
||||
|
||||
@dataclass(frozen=True, kw_only=True)
|
||||
class MLAAttentionSpec(FullAttentionSpec):
|
||||
# TODO(Lucas/Chen): less hacky way to do this
|
||||
@@ -420,6 +388,8 @@ class MLAAttentionSpec(FullAttentionSpec):
|
||||
model_version: str | None = None
|
||||
# Marks draft groups that flatten a non-causal query block into decode rows.
|
||||
non_causal_multi_token_decode: bool = False
|
||||
# MLA stores a single latent vector per state; there is no separate V.
|
||||
head_size_v: int = 0
|
||||
|
||||
def __post_init__(self):
|
||||
super().__post_init__()
|
||||
@@ -429,27 +399,6 @@ class MLAAttentionSpec(FullAttentionSpec):
|
||||
def storage_block_size(self) -> int:
|
||||
return self.block_size // self.compress_ratio
|
||||
|
||||
@property
|
||||
def real_page_size_bytes(self) -> int:
|
||||
if self.cache_dtype_str == "fp8_ds_mla":
|
||||
if self.model_version == "deepseek_v4":
|
||||
# DeepseekV4: 448B NoPE + 128B RoPE + 8B fp8 scale = 584B per token.
|
||||
# head_size stays semantic (512); bytes are determined here.
|
||||
return self.storage_block_size * 584
|
||||
# V3.2 main MLA: 656-byte custom layout (kv_lora_rank=512 +
|
||||
# qk_rope_head_dim=64, head_size=576). See flashmla_sparse.py.
|
||||
return self.block_size * 656
|
||||
if self.kv_quant_mode == KVQuantMode.INT4_PER_TOKEN_HEAD:
|
||||
head_dim = self.head_size // 2
|
||||
else:
|
||||
head_dim = self.head_size
|
||||
return (
|
||||
self.storage_block_size
|
||||
* self.num_kv_heads
|
||||
* head_dim
|
||||
* get_dtype_size(self.dtype)
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def merge(cls, specs: list[Self]) -> Self:
|
||||
assert all(isinstance(spec, MLAAttentionSpec) for spec in specs), (
|
||||
@@ -476,6 +425,8 @@ class MLAAttentionSpec(FullAttentionSpec):
|
||||
dtype=specs[0].dtype,
|
||||
kv_quant_mode=specs[0].kv_quant_mode,
|
||||
page_size_padded=specs[0].page_size_padded,
|
||||
num_head_slots=specs[0].num_head_slots,
|
||||
state_content_bytes=specs[0].state_content_bytes,
|
||||
indexes_kv_by_block_stride=block_stride_set.pop(),
|
||||
cache_dtype_str=cache_dtype_str_set.pop(),
|
||||
compress_ratio=compress_ratio_set.pop(),
|
||||
@@ -533,6 +484,8 @@ class RSWASpec(FullAttentionSpec):
|
||||
kv_quant_mode=base.kv_quant_mode,
|
||||
page_size_padded=base.page_size_padded,
|
||||
indexes_kv_by_block_stride=base.indexes_kv_by_block_stride,
|
||||
num_head_slots=base.num_head_slots,
|
||||
state_content_bytes=base.state_content_bytes,
|
||||
sliding_window=base.sliding_window,
|
||||
attention_chunk_size=base.attention_chunk_size,
|
||||
non_causal=base.non_causal,
|
||||
@@ -583,7 +536,6 @@ class ChunkedLocalAttentionSpec(AttentionSpec):
|
||||
@dataclass(frozen=True, kw_only=True)
|
||||
class SlidingWindowSpec(AttentionSpec):
|
||||
sliding_window: int
|
||||
head_size_v: int = None # type: ignore[assignment]
|
||||
# The trailing edge of the window is extended by ``extra_retained_tokens``
|
||||
# so that those extra trailing tokens' blocks are retained (but not
|
||||
# attended). This is needed for multi-module spec decoding which can
|
||||
@@ -591,30 +543,6 @@ class SlidingWindowSpec(AttentionSpec):
|
||||
# of the sequence, and thus needs to delay freeing/caching of blocks.
|
||||
extra_retained_tokens: int = 0
|
||||
|
||||
def __post_init__(self):
|
||||
if self.head_size_v is None:
|
||||
object.__setattr__(self, "head_size_v", self.head_size)
|
||||
|
||||
@property
|
||||
def real_page_size_bytes(self) -> int:
|
||||
# Mirror ``FullAttentionSpec.real_page_size_bytes`` for NVFP4 KV cache.
|
||||
if self.kv_quant_mode.is_nvfp4:
|
||||
last_dim = nvfp4_kv_cache_full_dim(
|
||||
self.head_size
|
||||
) + nvfp4_kv_cache_full_dim(self.head_size_v)
|
||||
return (
|
||||
self.block_size
|
||||
* self.num_kv_heads
|
||||
* last_dim
|
||||
* get_dtype_size(self.dtype)
|
||||
)
|
||||
return (
|
||||
self.block_size
|
||||
* self.num_kv_heads
|
||||
* (self.head_size + self.head_size_v)
|
||||
* get_dtype_size(self.dtype)
|
||||
)
|
||||
|
||||
def max_admission_blocks_per_request(
|
||||
self, max_in_flight_tokens: int, max_model_len: int
|
||||
) -> int:
|
||||
@@ -672,31 +600,20 @@ class SlidingWindowMLASpec(SlidingWindowSpec):
|
||||
alignment: int | None = None # Default to None for no padding.
|
||||
compress_ratio: int = 1
|
||||
model_version: str | None = None
|
||||
# MLA stores a single latent vector per state; there is no separate V.
|
||||
head_size_v: int = 0
|
||||
|
||||
def __post_init__(self):
|
||||
assert self.model_version in (None, "deepseek_v4"), (
|
||||
f"Unsupported model version: {self.model_version}"
|
||||
)
|
||||
super().__post_init__()
|
||||
_apply_alignment_padding(self)
|
||||
|
||||
@property
|
||||
def storage_block_size(self) -> int:
|
||||
return self.block_size // self.compress_ratio
|
||||
|
||||
@property
|
||||
def real_page_size_bytes(self) -> int:
|
||||
if self.model_version == "deepseek_v4" and self.cache_dtype_str == "fp8_ds_mla":
|
||||
# DeepseekV4 FlashMLA: 448B NoPE + 128B RoPE + 8B fp8 scale = 584B
|
||||
# per token. FlashInfer's contiguous bf16/fp8 cache falls through to
|
||||
# the element-size formula below.
|
||||
return self.storage_block_size * 584
|
||||
assert self.model_version in (None, "deepseek_v4"), (
|
||||
f"Unsupported model version: {self.model_version}"
|
||||
)
|
||||
return (
|
||||
self.storage_block_size
|
||||
* self.num_kv_heads
|
||||
* self.head_size
|
||||
* get_dtype_size(self.dtype)
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def merge(cls, specs: list[Self]) -> Self:
|
||||
assert all(isinstance(spec, SlidingWindowMLASpec) for spec in specs), (
|
||||
@@ -727,6 +644,8 @@ class SlidingWindowMLASpec(SlidingWindowSpec):
|
||||
head_size=specs[0].head_size,
|
||||
dtype=specs[0].dtype,
|
||||
page_size_padded=specs[0].page_size_padded,
|
||||
num_head_slots=specs[0].num_head_slots,
|
||||
state_content_bytes=specs[0].state_content_bytes,
|
||||
indexes_kv_by_block_stride=block_stride_set.pop(),
|
||||
sliding_window=sliding_window_set.pop(),
|
||||
extra_retained_tokens=extra_retained_set.pop(),
|
||||
@@ -853,6 +772,8 @@ class SinkFullAttentionSpec(FullAttentionSpec):
|
||||
kv_quant_mode=specs[0].kv_quant_mode,
|
||||
page_size_padded=specs[0].page_size_padded,
|
||||
indexes_kv_by_block_stride=specs[0].indexes_kv_by_block_stride,
|
||||
num_head_slots=specs[0].num_head_slots,
|
||||
state_content_bytes=specs[0].state_content_bytes,
|
||||
sliding_window=cls.merge_window_sizes(sliding_window),
|
||||
attention_chunk_size=cls.merge_window_sizes(attention_chunk_size),
|
||||
non_causal=any(spec.non_causal for spec in specs),
|
||||
|
||||
@@ -75,6 +75,7 @@ def get_kv_cache_spec(vllm_config: VllmConfig) -> dict[str, KVCacheSpec]:
|
||||
with set_current_vllm_config(vllm_config):
|
||||
indexes = backend.indexes_kv_by_block_stride()
|
||||
spec = replace(spec, indexes_kv_by_block_stride=indexes)
|
||||
spec = backend.customize_spec(spec)
|
||||
kv_cache_spec[layer_name] = spec
|
||||
return kv_cache_spec
|
||||
|
||||
|
||||
@@ -7925,6 +7925,7 @@ class GPUModelRunner(
|
||||
with set_current_vllm_config(self.vllm_config):
|
||||
indexes = backend.indexes_kv_by_block_stride()
|
||||
spec = replace(spec, indexes_kv_by_block_stride=indexes)
|
||||
spec = backend.customize_spec(spec)
|
||||
kv_cache_spec[layer_name] = spec
|
||||
|
||||
return kv_cache_spec
|
||||
|
||||
Reference in New Issue
Block a user