[Core][DSV4] Compact MXFP4 indexer KV cache and packed group overlays (#48993)

This commit is contained in:
Summer Yang
2026-07-22 16:58:44 -07:00
committed by GitHub
parent 149daf0d72
commit f3a920a076
3 changed files with 213 additions and 81 deletions
+11 -5
View File
@@ -26,6 +26,7 @@ from vllm.models.deepseek_v4.common.ops import (
fused_indexer_q_rope_quant,
fused_q_kv_rmsnorm,
)
from vllm.models.deepseek_v4.common.ops.fused_indexer_q import MXFP4_BLOCK_SIZE
if TYPE_CHECKING:
from vllm.v1.attention.backends.mla.sparse_swa import (
@@ -727,11 +728,16 @@ class DeepseekV4Indexer(nn.Module):
)
assert cache_config is not None, "Deepseek V4 indexer requires cache_config"
# NOTE(yifan): FP8 indxer cache use the same layout as V3.2:
# head_dim bytes = 128 fp8 + 4 fp32 scale = 132.
# For FP4 indexer cache, we still allocate the same amount of memory as FP8,
# but only use the first half of the memory.
k_cache_head_dim = self.head_dim + self.head_dim // self.quant_block_size * 4
if self.use_fp4_kv:
# MXFP4 stores two values per byte plus one UE8M0 byte per 32 values.
# head_dim bytes = 64 packed values + 4 UE8M0 scales = 68.
k_cache_head_dim = self.head_dim // 2 + self.head_dim // MXFP4_BLOCK_SIZE
else:
# NOTE(yifan): FP8 indexer cache uses the same layout as V3.2:
# head_dim bytes = 128 fp8 + 4 fp32 scale = 132.
k_cache_head_dim = (
self.head_dim + self.head_dim // self.quant_block_size * 4
)
self.k_cache = DeepseekV4IndexerCache(
head_dim=k_cache_head_dim,
dtype=torch.uint8,