[Config] Unify indexer cache dtype under attention_config.indexer_kv_dtype (#52550)

Signed-off-by: Yongye Zhu <zyy1102000@gmail.com>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Yongye Zhu
2026-08-17 10:20:04 +08:00
committed by GitHub
co-authored by Claude Opus 5
parent 0ad04cff1b
commit 967e104fad
6 changed files with 65 additions and 24 deletions
@@ -15,7 +15,7 @@ server_args: >-
--block-size 256 --block-size 256
--gpu-memory-utilization 0.5 --gpu-memory-utilization 0.5
--kv-cache-dtype fp8 --kv-cache-dtype fp8
--attention_config.use_fp4_indexer_cache=True --attention_config.indexer_kv_dtype=mxfp4
--max-num-batched-tokens 16384 --max-num-batched-tokens 16384
--max-num-seqs 128 --max-num-seqs 128
--speculative-config '{"method":"dspark", --speculative-config '{"method":"dspark",
@@ -2,4 +2,4 @@ model_name: "deepseek-ai/DeepSeek-V4-Flash"
accuracy_threshold: 0.95 accuracy_threshold: 0.95
num_questions: 1319 num_questions: 1319
num_fewshot: 5 num_fewshot: 5
server_args: "--trust-remote-code --kv-cache-dtype fp8 --block-size 256 --enable-expert-parallel --tensor-parallel-size 2 --attention_config.use_fp4_indexer_cache=True --moe-backend deep_gemm_mega_moe --tokenizer-mode deepseek_v4 --tool-call-parser deepseek_v4 --enable-auto-tool-choice --reasoning-parser deepseek_v4 --speculative_config.method=mtp --speculative_config.num_speculative_tokens=2" server_args: "--trust-remote-code --kv-cache-dtype fp8 --block-size 256 --enable-expert-parallel --tensor-parallel-size 2 --attention_config.indexer_kv_dtype=mxfp4 --moe-backend deep_gemm_mega_moe --tokenizer-mode deepseek_v4 --tool-call-parser deepseek_v4 --enable-auto-tool-choice --reasoning-parser deepseek_v4 --speculative_config.method=mtp --speculative_config.num_speculative_tokens=2"
+34 -7
View File
@@ -7,10 +7,13 @@ from typing import Any, Literal
from pydantic import field_validator from pydantic import field_validator
from vllm.config.utils import config from vllm.config.utils import config
from vllm.logger import init_logger
from vllm.v1.attention.backends.mla.prefill.registry import MLAPrefillBackendEnum from vllm.v1.attention.backends.mla.prefill.registry import MLAPrefillBackendEnum
from vllm.v1.attention.backends.registry import AttentionBackendEnum from vllm.v1.attention.backends.registry import AttentionBackendEnum
IndexerKVDType = Literal["bf16", "fp8", "mxfp4", "nvfp4"] logger = init_logger(__name__)
IndexerKVDType = Literal["auto", "bf16", "fp8", "mxfp4", "nvfp4"]
MiniMaxM3MSADecodeBackend = Literal["triton", "cutlass"] MiniMaxM3MSADecodeBackend = Literal["triton", "cutlass"]
@@ -65,12 +68,15 @@ class AttentionConfig:
use_prefill_query_quantization: bool = False use_prefill_query_quantization: bool = False
"""If set, quantize query for attention in prefill.""" """If set, quantize query for attention in prefill."""
use_fp4_indexer_cache: bool = False use_fp4_indexer_cache: bool | None = None
"""If set, use fp4 indexer cache for dsv32 family model (not support yet)""" """Deprecated alias for `indexer_kv_dtype`; use that instead. True maps to
`mxfp4`, False is a no-op (it selected the model default already)."""
indexer_kv_dtype: IndexerKVDType = "bf16" indexer_kv_dtype: IndexerKVDType = "auto"
"""Data type for the sparse-attention indexer K cache. Quantized formats """Data type for the sparse-attention indexer K cache. "auto" picks the
(fp8, mxfp4, nvfp4) require indexer kernel support in the backend.""" model's default (bf16 for MiniMax M3, fp8 for the DeepSeek sparse
indexer). Quantized formats (fp8, mxfp4, nvfp4) require indexer kernel
support in the backend."""
use_non_causal: bool = False use_non_causal: bool = False
"""Whether to use non-causal (bidirectional) attention.""" """Whether to use non-causal (bidirectional) attention."""
@@ -114,6 +120,26 @@ class AttentionConfig:
# layers still use the platform's normal automatic backend. # layers still use the platform's normal automatic backend.
self.backend = None self.backend = None
if self.use_fp4_indexer_cache is not None:
logger.warning(
"use_fp4_indexer_cache is deprecated and will be removed in "
"v0.19. Use indexer_kv_dtype instead (True -> 'mxfp4')."
)
if self.use_fp4_indexer_cache:
if self.indexer_kv_dtype not in ("auto", "mxfp4"):
raise ValueError(
"use_fp4_indexer_cache=True conflicts with "
f"indexer_kv_dtype={self.indexer_kv_dtype!r}. Set only "
"indexer_kv_dtype."
)
self.indexer_kv_dtype = "mxfp4"
def resolve_indexer_kv_dtype(self, default: IndexerKVDType) -> IndexerKVDType:
"""Resolve `indexer_kv_dtype`, substituting `default` for "auto"."""
if self.indexer_kv_dtype == "auto":
return default
return self.indexer_kv_dtype
def compute_hash(self) -> str: def compute_hash(self) -> str:
""" """
Provide a hash that uniquely identifies all the configs Provide a hash that uniquely identifies all the configs
@@ -124,7 +150,8 @@ class AttentionConfig:
""" """
from vllm.config.utils import get_hash_factors, hash_factors from vllm.config.utils import get_hash_factors, hash_factors
ignored_factors: set[str] = set() # Folded into indexer_kv_dtype by __post_init__.
ignored_factors: set[str] = {"use_fp4_indexer_cache"}
factors = get_hash_factors(self, ignored_factors) factors = get_hash_factors(self, ignored_factors)
return hash_factors(factors) return hash_factors(factors)
+2 -1
View File
@@ -56,6 +56,7 @@ from vllm.utils.multi_stream_utils import (
from vllm.v1.attention.backend import AttentionBackend, AttentionMetadata from vllm.v1.attention.backend import AttentionBackend, AttentionMetadata
from vllm.v1.attention.backends.mla.indexer import ( from vllm.v1.attention.backends.mla.indexer import (
DeepseekV4IndexerBackend, DeepseekV4IndexerBackend,
dsa_indexer_uses_fp4,
get_max_prefill_buffer_size, get_max_prefill_buffer_size,
) )
from vllm.v1.attention.backends.mla.sparse_swa import DeepseekV4SWACache from vllm.v1.attention.backends.mla.sparse_swa import DeepseekV4SWACache
@@ -805,7 +806,7 @@ class DeepseekV4Indexer(nn.Module):
self.q_lora_rank = q_lora_rank # 1536 self.q_lora_rank = q_lora_rank # 1536
self.compress_ratio = compress_ratio self.compress_ratio = compress_ratio
self.eager_scratch_pool = eager_scratch_pool self.eager_scratch_pool = eager_scratch_pool
self.use_fp4_kv = self.vllm_config.attention_config.use_fp4_indexer_cache self.use_fp4_kv = dsa_indexer_uses_fp4(vllm_config)
logger.info_once( logger.info_once(
"Using %s indexer cache for Lightning Indexer.", "Using %s indexer cache for Lightning Indexer.",
"MXFP4" if self.use_fp4_kv else "FP8", "MXFP4" if self.use_fp4_kv else "FP8",
+3 -1
View File
@@ -498,7 +498,9 @@ class MiniMaxM3SparseAttention(nn.Module, AttentionLayerBase):
set_default_quant_scales(self, register_buffer=True) set_default_quant_scales(self, register_buffer=True)
# Indexer side-cache dtype, mirroring --kv-cache-dtype for the main # Indexer side-cache dtype, mirroring --kv-cache-dtype for the main
# cache (--attention-config '{"indexer_kv_dtype": ...}'). # cache (--attention-config '{"indexer_kv_dtype": ...}').
self.indexer_kv_dtype = vllm_config.attention_config.indexer_kv_dtype self.indexer_kv_dtype = vllm_config.attention_config.resolve_indexer_kv_dtype(
"bf16"
)
# Shared top-k buffer: the indexer writes the selected blocks into it and # Shared top-k buffer: the indexer writes the selected blocks into it and
# the attend impl reads them back (so nothing crosses the eager break as a # the attend impl reads them back (so nothing crosses the eager break as a
+24 -13
View File
@@ -40,6 +40,28 @@ from vllm.v1.kv_cache_interface import KVCacheSpec, MLAAttentionSpec
logger = init_logger(__name__) logger = init_logger(__name__)
# The DSA indexer K cache is always quantized; "auto" means fp8 (V3.2 layout)
# and mxfp4 is the opt-in Blackwell path.
DSA_INDEXER_KV_DTYPES = ("fp8", "mxfp4")
def dsa_indexer_uses_fp4(vllm_config: VllmConfig) -> bool:
"""Whether the DeepSeek sparse indexer should use the MXFP4 K cache."""
kv_dtype = vllm_config.attention_config.resolve_indexer_kv_dtype("fp8")
if kv_dtype not in DSA_INDEXER_KV_DTYPES:
raise ValueError(
f"indexer_kv_dtype={kv_dtype!r} is not supported by the DeepSeek "
f"sparse indexer (expected one of {DSA_INDEXER_KV_DTYPES})."
)
use_fp4 = kv_dtype == "mxfp4"
if use_fp4 and not current_platform.is_device_capability_family(100):
raise ValueError(
"indexer_kv_dtype='mxfp4' requires Blackwell datacenter GPUs "
"(sm_10x, e.g. B200/GB200); sm_120 (consumer Blackwell) and "
"earlier architectures are not supported."
)
return use_fp4
@triton.jit @triton.jit
def _prepare_uniform_decode_kernel( def _prepare_uniform_decode_kernel(
@@ -526,18 +548,7 @@ class DeepseekV32IndexerMetadataBuilder(AttentionMetadataBuilder):
if self.vllm_config.speculative_config if self.vllm_config.speculative_config
else 0 else 0
) )
self.use_fp4_indexer_cache = ( self.use_fp4_indexer_cache = dsa_indexer_uses_fp4(self.vllm_config)
self.vllm_config.attention_config.use_fp4_indexer_cache
)
assert (
current_platform.is_device_capability_family(100)
or not self.use_fp4_indexer_cache
), (
"use_fp4_indexer_cache requires Blackwell datacenter GPUs "
"(sm_10x, e.g. B200/GB200); sm_120 (consumer Blackwell) and "
"earlier architectures are not supported."
)
next_n = self.num_speculative_tokens + 1 next_n = self.num_speculative_tokens + 1
self.decode_threshold = next_n self.decode_threshold = next_n
@@ -546,7 +557,7 @@ class DeepseekV32IndexerMetadataBuilder(AttentionMetadataBuilder):
self.supports_varlen = _supports_varlen_paged_mqa_logits() self.supports_varlen = _supports_varlen_paged_mqa_logits()
logger.info_once( logger.info_once(
"DSA indexer decode path: use_flattening=%s supports_varlen=%s " "DSA indexer decode path: use_flattening=%s supports_varlen=%s "
"(next_n=%d, use_fp4_indexer_cache=%s)", "(next_n=%d, use_fp4_cache=%s)",
self.use_flattening, self.use_flattening,
self.supports_varlen, self.supports_varlen,
next_n, next_n,