[Config] Unify indexer cache dtype under attention_config.indexer_kv_dtype (#52550)
Signed-off-by: Yongye Zhu <zyy1102000@gmail.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5
parent
0ad04cff1b
commit
967e104fad
@@ -56,6 +56,7 @@ from vllm.utils.multi_stream_utils import (
|
||||
from vllm.v1.attention.backend import AttentionBackend, AttentionMetadata
|
||||
from vllm.v1.attention.backends.mla.indexer import (
|
||||
DeepseekV4IndexerBackend,
|
||||
dsa_indexer_uses_fp4,
|
||||
get_max_prefill_buffer_size,
|
||||
)
|
||||
from vllm.v1.attention.backends.mla.sparse_swa import DeepseekV4SWACache
|
||||
@@ -805,7 +806,7 @@ class DeepseekV4Indexer(nn.Module):
|
||||
self.q_lora_rank = q_lora_rank # 1536
|
||||
self.compress_ratio = compress_ratio
|
||||
self.eager_scratch_pool = eager_scratch_pool
|
||||
self.use_fp4_kv = self.vllm_config.attention_config.use_fp4_indexer_cache
|
||||
self.use_fp4_kv = dsa_indexer_uses_fp4(vllm_config)
|
||||
logger.info_once(
|
||||
"Using %s indexer cache for Lightning Indexer.",
|
||||
"MXFP4" if self.use_fp4_kv else "FP8",
|
||||
|
||||
@@ -498,7 +498,9 @@ class MiniMaxM3SparseAttention(nn.Module, AttentionLayerBase):
|
||||
set_default_quant_scales(self, register_buffer=True)
|
||||
# Indexer side-cache dtype, mirroring --kv-cache-dtype for the main
|
||||
# cache (--attention-config '{"indexer_kv_dtype": ...}').
|
||||
self.indexer_kv_dtype = vllm_config.attention_config.indexer_kv_dtype
|
||||
self.indexer_kv_dtype = vllm_config.attention_config.resolve_indexer_kv_dtype(
|
||||
"bf16"
|
||||
)
|
||||
|
||||
# Shared top-k buffer: the indexer writes the selected blocks into it and
|
||||
# the attend impl reads them back (so nothing crosses the eager break as a
|
||||
|
||||
Reference in New Issue
Block a user