diff --git a/tests/quantization/test_turboquant.py b/tests/quantization/test_turboquant.py index f4880abcb5..04a365b48d 100644 --- a/tests/quantization/test_turboquant.py +++ b/tests/quantization/test_turboquant.py @@ -281,7 +281,10 @@ class TestTurboQuantKVCacheSpec: @pytest.mark.parametrize("preset", ALL_PRESETS) def test_kv_cache_spec_sets_kv_quant_mode(self, preset): from vllm.model_executor.layers.attention.attention import Attention - from vllm.v1.kv_cache_interface import KVQuantMode, TQFullAttentionSpec + from vllm.v1.attention.backends.turboquant_attn import ( + TurboQuantAttentionBackend, + ) + from vllm.v1.kv_cache_interface import FullAttentionSpec layer = SimpleNamespace( attn_type="decoder", @@ -294,10 +297,16 @@ class TestTurboQuantKVCacheSpec: ) vllm_config = SimpleNamespace(cache_config=SimpleNamespace(block_size=32)) + # The layer builds an unpacked spec; the worker's spec-collection + # loop applies TQ slot packing via the backend's customize_spec hook. spec = Attention.get_kv_cache_spec(layer, vllm_config) + assert isinstance(spec, FullAttentionSpec) + assert spec.kv_quant_mode.is_turboquant + assert spec.state_content_bytes is None - assert isinstance(spec, TQFullAttentionSpec) - assert spec.kv_quant_mode == KVQuantMode.TURBOQUANT + spec = TurboQuantAttentionBackend.customize_spec(spec) + expected_slot = TurboQuantConfig.from_cache_dtype(preset, 128).slot_size_aligned + assert spec.state_content_bytes == expected_slot class TestTurboQuantWorkspaceReservation: @@ -333,15 +342,15 @@ class TestTurboQuantWorkspaceReservation: @staticmethod def _fake_kv_cache_spec(): - from vllm.v1.kv_cache_interface import TQFullAttentionSpec + from vllm.v1.kv_cache_interface import FullAttentionSpec - return TQFullAttentionSpec( + return FullAttentionSpec( block_size=32, num_kv_heads=4, head_size=128, head_size_v=128, dtype=torch.uint8, - tq_slot_size=102, + state_content_bytes=102, ) def test_metadata_builder_reserves_decode_and_continuation_prefill_workspace( diff --git a/tests/v1/core/test_kv_cache_utils.py b/tests/v1/core/test_kv_cache_utils.py index 33430ce850..9b2afeb001 100644 --- a/tests/v1/core/test_kv_cache_utils.py +++ b/tests/v1/core/test_kv_cache_utils.py @@ -2766,21 +2766,20 @@ def test_unify_kv_cache_page_size_padding_requires_backend_support(): kv_cache_utils.unify_kv_cache_spec_page_size(specs) -def test_unpadded_page_size_without_quant_matches_real_page(): - # Without quantization the offload transfer width is just the raw page. - spec = new_kv_cache_spec() - assert spec.unpadded_page_size_bytes == spec.real_page_size_bytes - assert spec.page_size_bytes == spec.unpadded_page_size_bytes - - def test_unpadded_page_size_includes_per_token_head_scales(): # Per-token-head quant carries inline fp32 scales that are carved from the - # raw KV allocation, so they must be budgeted into the offload width. - spec = new_kv_cache_spec( - dtype=torch.uint8, kv_quant_mode=KVQuantMode.FP8_PER_TOKEN_HEAD + # raw KV allocation, so they must be budgeted into the offload width. The + # packing is published by the owning backend's customize_spec hook. + from vllm.v1.attention.backends.triton_attn import TritonAttentionBackend + + dense = new_kv_cache_spec(dtype=torch.uint8) + spec = TritonAttentionBackend.customize_spec( + new_kv_cache_spec( + dtype=torch.uint8, kv_quant_mode=KVQuantMode.FP8_PER_TOKEN_HEAD + ) ) scales = 2 * spec.block_size * spec.num_kv_heads * 4 - assert spec.unpadded_page_size_bytes == spec.real_page_size_bytes + scales + assert spec.unpadded_page_size_bytes == dense.unpadded_page_size_bytes + scales assert spec.page_size_bytes == spec.unpadded_page_size_bytes diff --git a/tests/v1/test_kv_cache_spec_registry.py b/tests/v1/test_kv_cache_spec_registry.py index 63834719c9..58c23ce285 100644 --- a/tests/v1/test_kv_cache_spec_registry.py +++ b/tests/v1/test_kv_cache_spec_registry.py @@ -34,7 +34,6 @@ from vllm.v1.kv_cache_interface import ( SinkFullAttentionSpec, SlidingWindowMLASpec, SlidingWindowSpec, - TQFullAttentionSpec, UniformTypeKVCacheSpecs, get_kv_cache_spec_kind, ) @@ -85,7 +84,6 @@ class _TrulyUnregisteredSpec(KVCacheSpec): spec_manager_map: dict[type[KVCacheSpec], type[SingleTypeKVCacheManager]] = { FullAttentionSpec: FullAttentionManager, - TQFullAttentionSpec: FullAttentionManager, MLAAttentionSpec: FullAttentionManager, HiddenStateCacheSpec: FullAttentionManager, SlidingWindowSpec: SlidingWindowManager, @@ -98,7 +96,6 @@ spec_manager_map: dict[type[KVCacheSpec], type[SingleTypeKVCacheManager]] = { spec_uniform_base_map: dict[type[KVCacheSpec], type[KVCacheSpec]] = { FullAttentionSpec: FullAttentionSpec, - TQFullAttentionSpec: FullAttentionSpec, MLAAttentionSpec: FullAttentionSpec, HiddenStateCacheSpec: FullAttentionSpec, SlidingWindowSpec: SlidingWindowSpec, @@ -113,13 +110,6 @@ spec_args_map: dict[type[KVCacheSpec], dict[str, Any]] = { FullAttentionSpec: dict( block_size=64, num_kv_heads=8, head_size=128, dtype=torch.bfloat16 ), - TQFullAttentionSpec: dict( - block_size=64, - num_kv_heads=8, - head_size=128, - dtype=torch.bfloat16, - tq_slot_size=256, - ), MLAAttentionSpec: dict( block_size=64, num_kv_heads=1, head_size=128, dtype=torch.bfloat16 ), @@ -262,7 +252,6 @@ class TestKVCacheSpecRegistry: def test_full_attention_family_specs_are_uniform(self): specs = [ make_spec(FullAttentionSpec), - make_spec(TQFullAttentionSpec), make_spec(MLAAttentionSpec), make_spec(HiddenStateCacheSpec), make_spec(SinkFullAttentionSpec), diff --git a/tests/v1/worker/test_dsv4_packed_zeroer_geometry.py b/tests/v1/worker/test_dsv4_packed_zeroer_geometry.py index 37f1b0737f..d5a698433c 100644 --- a/tests/v1/worker/test_dsv4_packed_zeroer_geometry.py +++ b/tests/v1/worker/test_dsv4_packed_zeroer_geometry.py @@ -93,6 +93,7 @@ def test_dsv4_packed_zeroer_geometry(): cache_dtype_str="fp8_ds_mla", alignment=576, model_version="deepseek_v4", + state_content_bytes=584, # >576 to allocate room for scales at back of page ) for _ in layer_names ] diff --git a/vllm/model_executor/layers/attention/attention.py b/vllm/model_executor/layers/attention/attention.py index e5960e68a4..b4831e2a0b 100644 --- a/vllm/model_executor/layers/attention/attention.py +++ b/vllm/model_executor/layers/attention/attention.py @@ -623,14 +623,17 @@ class Attention(nn.Module, AttentionLayerBase): # bytes per block. Otherwise (page_size_padded is None) the smallest # block is fine — ``unify`` scales it up by an integer ratio. shared_page = vllm_config.cache_config.skip_page_size_padded - sw_per_token = SlidingWindowSpec( - block_size=1, - num_kv_heads=self.num_kv_heads, - head_size=self.head_size, - head_size_v=self.head_size_v, - dtype=self.kv_cache_torch_dtype, - kv_quant_mode=quant_mode, - sliding_window=self.sliding_window, + # The backend owns its packing + sw_per_token = self.attn_backend.customize_spec( + SlidingWindowSpec( + block_size=1, + num_kv_heads=self.num_kv_heads, + head_size=self.head_size, + head_size_v=self.head_size_v, + dtype=self.kv_cache_torch_dtype, + kv_quant_mode=quant_mode, + sliding_window=self.sliding_window, + ) ).real_page_size_bytes sw_block_size = _largest_kernel_block_within( self.attn_backend, @@ -649,24 +652,6 @@ class Attention(nn.Module, AttentionLayerBase): sliding_window=self.sliding_window, page_size_padded=shared_page, ) - elif self.kv_cache_dtype.startswith("turboquant_"): - from vllm.model_executor.layers.quantization.turboquant.config import ( - TurboQuantConfig, - ) - from vllm.v1.kv_cache_interface import TQFullAttentionSpec - - tq_config = TurboQuantConfig.from_cache_dtype( - self.kv_cache_dtype, self.head_size - ) - return TQFullAttentionSpec( - block_size=block_size, - num_kv_heads=self.num_kv_heads, - head_size=self.head_size, - head_size_v=self.head_size, - dtype=self.kv_cache_torch_dtype, - kv_quant_mode=quant_mode, - tq_slot_size=tq_config.slot_size_aligned, - ) else: return FullAttentionSpec( block_size=block_size, diff --git a/vllm/model_executor/layers/attention/mla_attention.py b/vllm/model_executor/layers/attention/mla_attention.py index c57bd0ea31..839f4266cd 100644 --- a/vllm/model_executor/layers/attention/mla_attention.py +++ b/vllm/model_executor/layers/attention/mla_attention.py @@ -204,7 +204,7 @@ import itertools import math from abc import abstractmethod from collections.abc import Callable -from dataclasses import dataclass +from dataclasses import dataclass, replace from enum import Enum from math import lcm from typing import ClassVar, Generic, TypeVar, cast @@ -270,6 +270,7 @@ from vllm.utils.torch_utils import ( _encode_layer_name, _resolve_layer_name, direct_register_custom_op, + get_dtype_size, is_quantized_kv_cache, kv_cache_dtype_str_to_dtype, np_to_pinned_tensor, @@ -298,6 +299,7 @@ from vllm.v1.attention.selector import get_attn_backend from vllm.v1.kv_cache_interface import ( AttentionSpec, KVCacheSpec, + KVQuantMode, MLAAttentionSpec, SlidingWindowMLASpec, get_kv_quant_mode, @@ -1152,6 +1154,9 @@ class MLAAttention(nn.Module, AttentionLayerBase): dtype=kv_cache_dtype, cache_dtype_str=self.kv_cache_dtype, kv_quant_mode=get_kv_quant_mode(self.kv_cache_dtype), + # fp8_ds_mla: 656-byte custom layout (kv_lora_rank=512 + + # qk_rope_head_dim=64, head_size=576). See flashmla_sparse.py. + state_content_bytes=656 if self.kv_cache_dtype == "fp8_ds_mla" else None, ) if self.sliding_window is not None: return SlidingWindowMLASpec( @@ -1370,6 +1375,20 @@ class _DecodeConcatQuantFP8(QuantFP8): class MLACommonBackend(AttentionBackend): + @classmethod + def customize_spec(cls, spec: "AttentionSpec") -> "AttentionSpec": + """Per-token-head modes pack an inline fp32 scale pair after the + latent data (single-sided: ``head_size_v == 0`` for MLA).""" + mode = spec.kv_quant_mode + if spec.state_content_bytes is not None or not mode.is_per_token_head: + return spec + head_size = spec.head_size + if mode == KVQuantMode.INT4_PER_TOKEN_HEAD: + head_size //= 2 + scale_bytes = get_dtype_size(torch.float32) + content = head_size * get_dtype_size(spec.dtype) + 2 * scale_bytes + return replace(spec, state_content_bytes=content) + @staticmethod def get_name() -> str: return "TRITON_MLA" diff --git a/vllm/models/deepseek_v4/attention.py b/vllm/models/deepseek_v4/attention.py index 5b85daba52..74a481e4b5 100644 --- a/vllm/models/deepseek_v4/attention.py +++ b/vllm/models/deepseek_v4/attention.py @@ -670,6 +670,9 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC): alignment=576 if uses_fp8_ds_mla_layout else 512, model_version="deepseek_v4", kv_quant_mode=get_kv_quant_mode(self.kv_cache_dtype), + # DeepseekV4: 448B NoPE + 128B RoPE + 8B fp8 scale = 584B per token; + # head_size stays semantic (512). + state_content_bytes=584 if uses_fp8_ds_mla_layout else None, ) diff --git a/vllm/models/kimi_k3/nvidia/mla.py b/vllm/models/kimi_k3/nvidia/mla.py index 17fd1f2658..71e9fa79f0 100644 --- a/vllm/models/kimi_k3/nvidia/mla.py +++ b/vllm/models/kimi_k3/nvidia/mla.py @@ -400,6 +400,8 @@ class MultiHeadLatentAttention(nn.Module, AttentionLayerBase): dtype=kv_cache_dtype, cache_dtype_str=self.kv_cache_dtype, kv_quant_mode=get_kv_quant_mode(self.kv_cache_dtype), + # fp8_ds_mla: 656-byte custom layout; see flashmla_sparse.py. + state_content_bytes=656 if self.kv_cache_dtype == "fp8_ds_mla" else None, non_causal_multi_token_decode=self.non_causal_multi_token_decode, ) diff --git a/vllm/platforms/interface.py b/vllm/platforms/interface.py index 939094d4c7..c3ade53ca1 100644 --- a/vllm/platforms/interface.py +++ b/vllm/platforms/interface.py @@ -687,13 +687,15 @@ class Platform: def per_token_page_bytes(dtype: "torch.dtype", cache_dtype: str) -> int: """Bytes one token occupies in one layer, for the given dtype.""" - return FullAttentionSpec( + spec = FullAttentionSpec( block_size=1, num_kv_heads=model_config.get_num_kv_heads(parallel_config), head_size=model_config.get_head_size(), dtype=dtype, kv_quant_mode=get_kv_quant_mode(cache_dtype), - ).page_size_bytes + ) + # The backend owns its packing + return backend_cls.customize_spec(spec).page_size_bytes primary_dtype = ( STR_DTYPE_TO_TORCH_DTYPE[cache_config.cache_dtype] @@ -809,23 +811,18 @@ class Platform: # when all attention layers are TQ. With mixed skip+TQ the skip # layers still use the standard layout — take max so mamba # padding covers the largest actual page. - from vllm.model_executor.layers.quantization.turboquant.config import ( - TurboQuantConfig, + from vllm.v1.attention.backends.turboquant_attn import ( + TurboQuantAttentionBackend, ) - from vllm.v1.kv_cache_interface import TQFullAttentionSpec - tq_cfg = TurboQuantConfig.from_cache_dtype( - cache_config.cache_dtype, model_config.get_head_size() - ) - tq_page = TQFullAttentionSpec( + tq_spec = FullAttentionSpec( block_size=1, num_kv_heads=model_config.get_num_kv_heads(parallel_config), head_size=model_config.get_head_size(), - head_size_v=model_config.get_head_size(), dtype=kv_cache_dtype, kv_quant_mode=kv_quant_mode, - tq_slot_size=tq_cfg.slot_size_aligned, - ).page_size_bytes + ) + tq_page = TurboQuantAttentionBackend.customize_spec(tq_spec).page_size_bytes if cache_config.kv_cache_dtype_skip_layers: skip_page = FullAttentionSpec( block_size=1, @@ -840,12 +837,15 @@ class Platform: else: attn_page_size_1_token = tq_page else: - attn_page_size_1_token = FullAttentionSpec( + attn_spec = FullAttentionSpec( block_size=1, num_kv_heads=model_config.get_num_kv_heads(parallel_config), head_size=model_config.get_head_size(), dtype=kv_cache_dtype, kv_quant_mode=kv_quant_mode, + ) + attn_page_size_1_token = backend_cls.customize_spec( + attn_spec ).page_size_bytes # Compute mamba page size diff --git a/vllm/v1/attention/backend.py b/vllm/v1/attention/backend.py index 12d19221fb..672f9e6376 100644 --- a/vllm/v1/attention/backend.py +++ b/vllm/v1/attention/backend.py @@ -24,7 +24,7 @@ if TYPE_CHECKING: from vllm.model_executor.layers.quantization.utils.quant_utils import QuantKey from vllm.platforms.interface import DeviceCapability from vllm.v1.attention.backends.utils import KVCacheLayoutType - from vllm.v1.kv_cache_interface import KVCacheSpec, KVQuantMode + from vllm.v1.kv_cache_interface import AttentionSpec, KVCacheSpec, KVQuantMode from vllm.v1.kv_cache_interface import get_kv_quant_mode @@ -197,6 +197,20 @@ class AttentionBackend(ABC): return True return False + @classmethod + def customize_spec(cls, spec: "AttentionSpec") -> "AttentionSpec": + """Adjust the layer's KV cache spec for this backend's kernels. Used when the + kernels want KV packed in a specific way. + + NOTE: temporary compatibility API. Today the Attention layer builds the spec + from the model config and the backend only gets to adjust it post-hoc; the end + state is for the backend to build and return the spec directly, at which point + this hook goes away. + + (see: https://github.com/vllm-project/vllm/issues/42449) + """ + return spec + @classmethod def get_preferred_block_size(cls, default_block_size: int) -> int: supported_sizes = cls.get_supported_kernel_block_sizes() diff --git a/vllm/v1/attention/backends/flashinfer.py b/vllm/v1/attention/backends/flashinfer.py index 187ee86189..2898e51c24 100755 --- a/vllm/v1/attention/backends/flashinfer.py +++ b/vllm/v1/attention/backends/flashinfer.py @@ -2,7 +2,7 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project """Attention layer with FlashInfer.""" -from dataclasses import dataclass +from dataclasses import dataclass, replace from enum import Enum from functools import partial from typing import ClassVar @@ -50,6 +50,7 @@ from vllm.utils.math_utils import cdiv from vllm.utils.torch_utils import ( PIN_MEMORY, canonicalize_singleton_dim_strides, + get_dtype_size, is_quantized_kv_cache, is_strictly_contiguous, nvfp4_kv_cache_full_dim, @@ -387,6 +388,21 @@ class BatchDCPPrefillWrapper: class FlashInferBackend(AttentionBackend): + @classmethod + def customize_spec(cls, spec: "AttentionSpec") -> "AttentionSpec": + """NVFP4 stores K and V as separate per-head slots of packed fp4 data + plus fp8 block scales.""" + if spec.state_content_bytes is not None or not spec.kv_quant_mode.is_nvfp4: + return spec + hs_k = nvfp4_kv_cache_full_dim(spec.head_size) + hs_v = nvfp4_kv_cache_full_dim(spec.head_size_v) + assert hs_k == hs_v, "nvfp4 with asymmetric K/V head sizes not yet supported" + return replace( + spec, + num_head_slots=2 * spec.num_kv_heads, + state_content_bytes=hs_k * get_dtype_size(spec.dtype), + ) + supported_dtypes: ClassVar[list[torch.dtype]] = [torch.float16, torch.bfloat16] supported_kv_cache_dtypes: ClassVar[list[CacheDType]] = [ "auto", diff --git a/vllm/v1/attention/backends/mla/sparse_swa.py b/vllm/v1/attention/backends/mla/sparse_swa.py index da0c125c62..f4e5874f9d 100644 --- a/vllm/v1/attention/backends/mla/sparse_swa.py +++ b/vllm/v1/attention/backends/mla/sparse_swa.py @@ -97,6 +97,8 @@ class DeepseekV4SWACache(torch.nn.Module, AttentionLayerBase): dtype=self.dtype, sliding_window=self.window_size, cache_dtype_str=self.cache_config.cache_dtype, + # DeepseekV4 fp8_ds_mla: 584B per token (448B NoPE + 128B RoPE + 8B scales) + state_content_bytes=584 if uses_fp8_ds_mla_layout else None, # 576B for FlashMLA packing; 512B for FlashInfer sparse (#44577). alignment=576 if uses_fp8_ds_mla_layout else 512, model_version="deepseek_v4", diff --git a/vllm/v1/attention/backends/triton_attn.py b/vllm/v1/attention/backends/triton_attn.py index 7814a8c0f8..4b1c167e7f 100644 --- a/vllm/v1/attention/backends/triton_attn.py +++ b/vllm/v1/attention/backends/triton_attn.py @@ -2,7 +2,7 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project """High-Performance Triton-only Attention layer.""" -from dataclasses import dataclass +from dataclasses import dataclass, replace from typing import ClassVar import torch @@ -19,7 +19,7 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import ( from vllm.platforms import current_platform from vllm.platforms.interface import DeviceCapability from vllm.utils.math_utils import next_power_of_2 -from vllm.utils.torch_utils import is_quantized_kv_cache +from vllm.utils.torch_utils import get_dtype_size, is_quantized_kv_cache from vllm.v1.attention.backend import ( AttentionBackend, AttentionCGSupport, @@ -274,6 +274,20 @@ class TritonAttentionMetadataBuilder(AttentionMetadataBuilder[TritonAttentionMet class TritonAttentionBackend(AttentionBackend): + @classmethod + def customize_spec(cls, spec: "AttentionSpec") -> "AttentionSpec": + """Per-token-head modes pack inline fp32 scales after each head's + data, so the content is (data + one scale) per K/V side.""" + mode = spec.kv_quant_mode + if spec.state_content_bytes is not None or not mode.is_per_token_head: + return spec + hs_k, hs_v = spec.head_size, spec.head_size_v + if mode == KVQuantMode.INT4_PER_TOKEN_HEAD: + hs_k, hs_v = hs_k // 2, hs_v // 2 + scale_bytes = get_dtype_size(torch.float32) + content = (hs_k + hs_v) * get_dtype_size(spec.dtype) + 2 * scale_bytes + return replace(spec, state_content_bytes=content) + supported_dtypes: ClassVar[list[torch.dtype]] = [ torch.float16, torch.bfloat16, diff --git a/vllm/v1/attention/backends/turboquant_attn.py b/vllm/v1/attention/backends/turboquant_attn.py index ab9e375b0a..5d743be886 100644 --- a/vllm/v1/attention/backends/turboquant_attn.py +++ b/vllm/v1/attention/backends/turboquant_attn.py @@ -19,7 +19,7 @@ Per-head per-position slot layout: import contextlib import functools import math -from dataclasses import dataclass +from dataclasses import dataclass, replace from typing import Any, ClassVar import torch @@ -137,6 +137,21 @@ class TurboQuantAttentionBackend(AttentionBackend): "turboquant_3bit_nc", ] + @classmethod + def customize_spec(cls, spec: AttentionSpec) -> AttentionSpec: + """TurboQuant packs K+V into one slot per head.""" + if spec.state_content_bytes is not None or not spec.kv_quant_mode.is_turboquant: + return spec + from vllm.model_executor.layers.quantization.turboquant.config import ( + TurboQuantConfig, + ) + + # KVQuantMode member names mirror the preset strings. + tq = TurboQuantConfig.from_cache_dtype( + spec.kv_quant_mode.name.lower(), spec.head_size + ) + return replace(spec, state_content_bytes=tq.slot_size_aligned) + @staticmethod def get_name() -> str: return "TURBOQUANT" diff --git a/vllm/v1/core/single_type_kv_cache_manager.py b/vllm/v1/core/single_type_kv_cache_manager.py index ded6bc809a..75acb7f497 100644 --- a/vllm/v1/core/single_type_kv_cache_manager.py +++ b/vllm/v1/core/single_type_kv_cache_manager.py @@ -28,7 +28,6 @@ from vllm.v1.kv_cache_interface import ( SinkFullAttentionSpec, SlidingWindowMLASpec, SlidingWindowSpec, - TQFullAttentionSpec, ) from vllm.v1.kv_cache_spec_registry import KVCacheSpecRegistry from vllm.v1.request import Request @@ -1929,11 +1928,6 @@ def register_all_kvcache_specs(vllm_config): ) # FullAttentionSpec subclasses — grouped with FullAttentionSpec - KVCacheSpecRegistry.register( - TQFullAttentionSpec, - FullAttentionManager, - uniform_type_base_spec=FullAttentionSpec, - ) KVCacheSpecRegistry.register( MLAAttentionSpec, FullAttentionManager, uniform_type_base_spec=FullAttentionSpec ) diff --git a/vllm/v1/kv_cache_interface.py b/vllm/v1/kv_cache_interface.py index 1ab3facb49..90391f6ef0 100644 --- a/vllm/v1/kv_cache_interface.py +++ b/vllm/v1/kv_cache_interface.py @@ -16,7 +16,7 @@ from typing_extensions import Self from vllm.logger import init_logger from vllm.utils.math_utils import cdiv, round_up -from vllm.utils.torch_utils import get_dtype_size, nvfp4_kv_cache_full_dim +from vllm.utils.torch_utils import get_dtype_size from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum from vllm.v1.kv_cache_spec_registry import KVCacheSpecRegistry @@ -46,7 +46,11 @@ class KVQuantMode(IntEnum): FP8_PER_TOKEN_HEAD = 3 # per-token-head dynamic scales for fp8 INT4_PER_TOKEN_HEAD = 4 # packed 2×int4/byte, RHT + asymmetric zp NVFP4 = 5 # packed fp4 data + fp8 block scales - TURBOQUANT = 6 # Hadamard-rotated Lloyd-Max quant, packed K+V per slot + # Hadamard-rotated Lloyd-Max quant, packed K+V per slot. + TURBOQUANT_K8V4 = 6 + TURBOQUANT_4BIT_NC = 7 + TURBOQUANT_K3V4_NC = 8 + TURBOQUANT_3BIT_NC = 9 @property def is_per_token_head(self) -> bool: @@ -64,8 +68,13 @@ class KVQuantMode(IntEnum): @property def is_turboquant(self) -> bool: - """True for turboquant quantization mode.""" - return self == KVQuantMode.TURBOQUANT + """True for any turboquant quantization mode.""" + return self in ( + KVQuantMode.TURBOQUANT_K8V4, + KVQuantMode.TURBOQUANT_4BIT_NC, + KVQuantMode.TURBOQUANT_K3V4_NC, + KVQuantMode.TURBOQUANT_3BIT_NC, + ) def get_kv_quant_mode(kv_cache_dtype: str) -> KVQuantMode: @@ -79,7 +88,7 @@ def get_kv_quant_mode(kv_cache_dtype: str) -> KVQuantMode: if kv_cache_dtype.startswith("nvfp4"): return KVQuantMode.NVFP4 if isinstance(kv_cache_dtype, str) and kv_cache_dtype.startswith("turboquant_"): - return KVQuantMode.TURBOQUANT + return KVQuantMode[kv_cache_dtype.upper()] if isinstance(kv_cache_dtype, str) and kv_cache_dtype.startswith("fp8"): return KVQuantMode.FP8_PER_TENSOR return KVQuantMode.NONE @@ -210,21 +219,37 @@ class AttentionSpec(KVCacheSpec): num_kv_heads: int head_size: int dtype: torch.dtype + head_size_v: int = None # type: ignore[assignment] kv_quant_mode: KVQuantMode = KVQuantMode.NONE page_size_padded: int | None = None indexes_kv_by_block_stride: bool = False + num_head_slots: int | None = None + """H of the logical ``[B, H, N, C]`` page when packing diverges from one + slot per KV head. None means one slot per KV head. Published by the backend. + """ + state_content_bytes: int | None = None + """C in bytes when packed; None means dense K/V content.""" + + def __post_init__(self): + if self.head_size_v is None: + object.__setattr__(self, "head_size_v", self.head_size) + + @property + def num_heads(self) -> int: + if self.num_head_slots is not None: + return self.num_head_slots + return self.num_kv_heads + + @property + def state_content_size_bytes(self) -> int: + """Bytes per (head slot, stored state) cell of the page.""" + if self.state_content_bytes is not None: + return self.state_content_bytes + return (self.head_size + self.head_size_v) * get_dtype_size(self.dtype) @property def unpadded_page_size_bytes(self) -> int: - unpadded = self.real_page_size_bytes - # Per-token-head scales are stored in separate tensors managed - # by the attention backend, but the memory is carved from the - # raw KV cache allocation so it must be budgeted here. - if self.kv_quant_mode.is_per_token_head: - unpadded += ( - 2 * self.block_size * self.num_kv_heads * get_dtype_size(torch.float32) - ) - return unpadded + return self.num_heads * self.storage_block_size * self.state_content_size_bytes @property def page_size_bytes(self) -> int: @@ -235,20 +260,10 @@ class AttentionSpec(KVCacheSpec): @property def real_page_size_bytes(self) -> int: - if self.kv_quant_mode.is_nvfp4: - # Packed layout: fp4 data + fp8 block scales per head. - head_dim = nvfp4_kv_cache_full_dim(self.head_size) - elif self.kv_quant_mode == KVQuantMode.INT4_PER_TOKEN_HEAD: - head_dim = self.head_size // 2 - else: - head_dim = self.head_size - return ( - 2 - * self.block_size - * self.num_kv_heads - * head_dim - * get_dtype_size(self.dtype) - ) + """Alias of ``unpadded_page_size_bytes`` + TODO(lucas): follow up with TPU backend to see if we can remove this property. + """ + return self.unpadded_page_size_bytes def max_num_blocks_per_req(self, vllm_config: VllmConfig, max_len: int) -> int: parallel_config = vllm_config.parallel_config @@ -267,8 +282,6 @@ class FullAttentionSpec(AttentionSpec): In this case, we use FullAttentionSpec and record the sliding window size. """ - head_size_v: int = None # type: ignore[assignment] - sliding_window: int | None = None """ Default to None for not using sliding window attention. @@ -284,10 +297,6 @@ class FullAttentionSpec(AttentionSpec): cache layout itself. """ - def __post_init__(self): - if self.head_size_v is None: - object.__setattr__(self, "head_size_v", self.head_size) - def max_memory_usage_bytes(self, vllm_config: VllmConfig) -> int: max_model_len = vllm_config.model_config.max_model_len dcp_world_size = vllm_config.parallel_config.decode_context_parallel_size @@ -337,6 +346,8 @@ class FullAttentionSpec(AttentionSpec): kv_quant_mode=specs[0].kv_quant_mode, page_size_padded=specs[0].page_size_padded, indexes_kv_by_block_stride=specs[0].indexes_kv_by_block_stride, + num_head_slots=specs[0].num_head_slots, + state_content_bytes=specs[0].state_content_bytes, sliding_window=cls.merge_window_sizes(sliding_window), attention_chunk_size=cls.merge_window_sizes(attention_chunk_size), # If any layer in the group is non-causal, treat the group as @@ -357,23 +368,6 @@ class FullAttentionSpec(AttentionSpec): ) return merged_spec - @property - def real_page_size_bytes(self) -> int: - if self.kv_quant_mode.is_nvfp4: - # Packed layout per head: fp4 data + fp8 block scales. - # fp4 data: head_size//2 bytes (2 fp4 values per byte) - # fp8 block scale: head_size//16 bytes (1 scale per 16 elements) - last_dim = nvfp4_kv_cache_full_dim( - self.head_size - ) + nvfp4_kv_cache_full_dim(self.head_size_v) - elif self.kv_quant_mode == KVQuantMode.INT4_PER_TOKEN_HEAD: - last_dim = self.head_size // 2 + self.head_size_v // 2 - else: - last_dim = self.head_size + self.head_size_v - return ( - self.block_size * self.num_kv_heads * last_dim * get_dtype_size(self.dtype) - ) - def _apply_alignment_padding(spec: MLAAttentionSpec | SlidingWindowMLASpec): if spec.alignment is None: @@ -384,32 +378,6 @@ def _apply_alignment_padding(spec: MLAAttentionSpec | SlidingWindowMLASpec): object.__setattr__(spec, "page_size_padded", padded_page_size) -@dataclass(frozen=True, kw_only=True) -class TQFullAttentionSpec(FullAttentionSpec): - """FullAttentionSpec with TQ-aware page size. - - Python equivalent of the C++ TQ4FullAttentionSpec. Overrides - real_page_size_bytes to use TQ slot bytes instead of the raw - head_size * dtype formula. - """ - - tq_slot_size: int = 0 - - @property - def real_page_size_bytes(self) -> int: - if self.tq_slot_size > 0: - return self.block_size * self.num_kv_heads * self.tq_slot_size - return super().real_page_size_bytes - - @classmethod - def merge(cls, specs: list[Self]) -> Self: - merged = super().merge(specs) - assert all(s.tq_slot_size == specs[0].tq_slot_size for s in specs), ( - "All TQ layers in the same KV cache group must use the same tq_slot_size." - ) - return replace(merged, tq_slot_size=specs[0].tq_slot_size) - - @dataclass(frozen=True, kw_only=True) class MLAAttentionSpec(FullAttentionSpec): # TODO(Lucas/Chen): less hacky way to do this @@ -420,6 +388,8 @@ class MLAAttentionSpec(FullAttentionSpec): model_version: str | None = None # Marks draft groups that flatten a non-causal query block into decode rows. non_causal_multi_token_decode: bool = False + # MLA stores a single latent vector per state; there is no separate V. + head_size_v: int = 0 def __post_init__(self): super().__post_init__() @@ -429,27 +399,6 @@ class MLAAttentionSpec(FullAttentionSpec): def storage_block_size(self) -> int: return self.block_size // self.compress_ratio - @property - def real_page_size_bytes(self) -> int: - if self.cache_dtype_str == "fp8_ds_mla": - if self.model_version == "deepseek_v4": - # DeepseekV4: 448B NoPE + 128B RoPE + 8B fp8 scale = 584B per token. - # head_size stays semantic (512); bytes are determined here. - return self.storage_block_size * 584 - # V3.2 main MLA: 656-byte custom layout (kv_lora_rank=512 + - # qk_rope_head_dim=64, head_size=576). See flashmla_sparse.py. - return self.block_size * 656 - if self.kv_quant_mode == KVQuantMode.INT4_PER_TOKEN_HEAD: - head_dim = self.head_size // 2 - else: - head_dim = self.head_size - return ( - self.storage_block_size - * self.num_kv_heads - * head_dim - * get_dtype_size(self.dtype) - ) - @classmethod def merge(cls, specs: list[Self]) -> Self: assert all(isinstance(spec, MLAAttentionSpec) for spec in specs), ( @@ -476,6 +425,8 @@ class MLAAttentionSpec(FullAttentionSpec): dtype=specs[0].dtype, kv_quant_mode=specs[0].kv_quant_mode, page_size_padded=specs[0].page_size_padded, + num_head_slots=specs[0].num_head_slots, + state_content_bytes=specs[0].state_content_bytes, indexes_kv_by_block_stride=block_stride_set.pop(), cache_dtype_str=cache_dtype_str_set.pop(), compress_ratio=compress_ratio_set.pop(), @@ -533,6 +484,8 @@ class RSWASpec(FullAttentionSpec): kv_quant_mode=base.kv_quant_mode, page_size_padded=base.page_size_padded, indexes_kv_by_block_stride=base.indexes_kv_by_block_stride, + num_head_slots=base.num_head_slots, + state_content_bytes=base.state_content_bytes, sliding_window=base.sliding_window, attention_chunk_size=base.attention_chunk_size, non_causal=base.non_causal, @@ -583,7 +536,6 @@ class ChunkedLocalAttentionSpec(AttentionSpec): @dataclass(frozen=True, kw_only=True) class SlidingWindowSpec(AttentionSpec): sliding_window: int - head_size_v: int = None # type: ignore[assignment] # The trailing edge of the window is extended by ``extra_retained_tokens`` # so that those extra trailing tokens' blocks are retained (but not # attended). This is needed for multi-module spec decoding which can @@ -591,30 +543,6 @@ class SlidingWindowSpec(AttentionSpec): # of the sequence, and thus needs to delay freeing/caching of blocks. extra_retained_tokens: int = 0 - def __post_init__(self): - if self.head_size_v is None: - object.__setattr__(self, "head_size_v", self.head_size) - - @property - def real_page_size_bytes(self) -> int: - # Mirror ``FullAttentionSpec.real_page_size_bytes`` for NVFP4 KV cache. - if self.kv_quant_mode.is_nvfp4: - last_dim = nvfp4_kv_cache_full_dim( - self.head_size - ) + nvfp4_kv_cache_full_dim(self.head_size_v) - return ( - self.block_size - * self.num_kv_heads - * last_dim - * get_dtype_size(self.dtype) - ) - return ( - self.block_size - * self.num_kv_heads - * (self.head_size + self.head_size_v) - * get_dtype_size(self.dtype) - ) - def max_admission_blocks_per_request( self, max_in_flight_tokens: int, max_model_len: int ) -> int: @@ -672,31 +600,20 @@ class SlidingWindowMLASpec(SlidingWindowSpec): alignment: int | None = None # Default to None for no padding. compress_ratio: int = 1 model_version: str | None = None + # MLA stores a single latent vector per state; there is no separate V. + head_size_v: int = 0 def __post_init__(self): + assert self.model_version in (None, "deepseek_v4"), ( + f"Unsupported model version: {self.model_version}" + ) + super().__post_init__() _apply_alignment_padding(self) @property def storage_block_size(self) -> int: return self.block_size // self.compress_ratio - @property - def real_page_size_bytes(self) -> int: - if self.model_version == "deepseek_v4" and self.cache_dtype_str == "fp8_ds_mla": - # DeepseekV4 FlashMLA: 448B NoPE + 128B RoPE + 8B fp8 scale = 584B - # per token. FlashInfer's contiguous bf16/fp8 cache falls through to - # the element-size formula below. - return self.storage_block_size * 584 - assert self.model_version in (None, "deepseek_v4"), ( - f"Unsupported model version: {self.model_version}" - ) - return ( - self.storage_block_size - * self.num_kv_heads - * self.head_size - * get_dtype_size(self.dtype) - ) - @classmethod def merge(cls, specs: list[Self]) -> Self: assert all(isinstance(spec, SlidingWindowMLASpec) for spec in specs), ( @@ -727,6 +644,8 @@ class SlidingWindowMLASpec(SlidingWindowSpec): head_size=specs[0].head_size, dtype=specs[0].dtype, page_size_padded=specs[0].page_size_padded, + num_head_slots=specs[0].num_head_slots, + state_content_bytes=specs[0].state_content_bytes, indexes_kv_by_block_stride=block_stride_set.pop(), sliding_window=sliding_window_set.pop(), extra_retained_tokens=extra_retained_set.pop(), @@ -853,6 +772,8 @@ class SinkFullAttentionSpec(FullAttentionSpec): kv_quant_mode=specs[0].kv_quant_mode, page_size_padded=specs[0].page_size_padded, indexes_kv_by_block_stride=specs[0].indexes_kv_by_block_stride, + num_head_slots=specs[0].num_head_slots, + state_content_bytes=specs[0].state_content_bytes, sliding_window=cls.merge_window_sizes(sliding_window), attention_chunk_size=cls.merge_window_sizes(attention_chunk_size), non_causal=any(spec.non_causal for spec in specs), diff --git a/vllm/v1/worker/gpu/attn_utils.py b/vllm/v1/worker/gpu/attn_utils.py index d2977870da..77d56ad882 100644 --- a/vllm/v1/worker/gpu/attn_utils.py +++ b/vllm/v1/worker/gpu/attn_utils.py @@ -75,6 +75,7 @@ def get_kv_cache_spec(vllm_config: VllmConfig) -> dict[str, KVCacheSpec]: with set_current_vllm_config(vllm_config): indexes = backend.indexes_kv_by_block_stride() spec = replace(spec, indexes_kv_by_block_stride=indexes) + spec = backend.customize_spec(spec) kv_cache_spec[layer_name] = spec return kv_cache_spec diff --git a/vllm/v1/worker/gpu_model_runner.py b/vllm/v1/worker/gpu_model_runner.py index 976d5df3bc..cd1679bdd9 100644 --- a/vllm/v1/worker/gpu_model_runner.py +++ b/vllm/v1/worker/gpu_model_runner.py @@ -7925,6 +7925,7 @@ class GPUModelRunner( with set_current_vllm_config(self.vllm_config): indexes = backend.indexes_kv_by_block_stride() spec = replace(spec, indexes_kv_by_block_stride=indexes) + spec = backend.customize_spec(spec) kv_cache_spec[layer_name] = spec return kv_cache_spec