[Core][DSV4] Compact MXFP4 indexer KV cache and packed group overlays (#48993)
This commit is contained in:
@@ -26,6 +26,7 @@ from vllm.models.deepseek_v4.common.ops import (
|
||||
fused_indexer_q_rope_quant,
|
||||
fused_q_kv_rmsnorm,
|
||||
)
|
||||
from vllm.models.deepseek_v4.common.ops.fused_indexer_q import MXFP4_BLOCK_SIZE
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.v1.attention.backends.mla.sparse_swa import (
|
||||
@@ -727,11 +728,16 @@ class DeepseekV4Indexer(nn.Module):
|
||||
)
|
||||
|
||||
assert cache_config is not None, "Deepseek V4 indexer requires cache_config"
|
||||
# NOTE(yifan): FP8 indxer cache use the same layout as V3.2:
|
||||
# head_dim bytes = 128 fp8 + 4 fp32 scale = 132.
|
||||
# For FP4 indexer cache, we still allocate the same amount of memory as FP8,
|
||||
# but only use the first half of the memory.
|
||||
k_cache_head_dim = self.head_dim + self.head_dim // self.quant_block_size * 4
|
||||
if self.use_fp4_kv:
|
||||
# MXFP4 stores two values per byte plus one UE8M0 byte per 32 values.
|
||||
# head_dim bytes = 64 packed values + 4 UE8M0 scales = 68.
|
||||
k_cache_head_dim = self.head_dim // 2 + self.head_dim // MXFP4_BLOCK_SIZE
|
||||
else:
|
||||
# NOTE(yifan): FP8 indexer cache uses the same layout as V3.2:
|
||||
# head_dim bytes = 128 fp8 + 4 fp32 scale = 132.
|
||||
k_cache_head_dim = (
|
||||
self.head_dim + self.head_dim // self.quant_block_size * 4
|
||||
)
|
||||
self.k_cache = DeepseekV4IndexerCache(
|
||||
head_dim=k_cache_head_dim,
|
||||
dtype=torch.uint8,
|
||||
|
||||
Reference in New Issue
Block a user