[Model][DSv4.1] Store the whole KV in MXFP8 (FlashMLA V4.1 record) (#56893)

Signed-off-by: Yongye Zhu <yongye@inferact.ai>
Signed-off-by: Yongye Zhu <zyy1102000@gmail.com>
Co-authored-by: Yongye Zhu <yongye@inferact.ai>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Yongye Zhu
2026-09-14 20:12:55 -07:00
committed by GitHub
co-authored by Yongye Zhu Claude Opus 5
parent a529c1a748
commit e6eb0d120c
14 changed files with 709 additions and 110 deletions
+34 -9
View File
@@ -51,6 +51,7 @@ from vllm.model_executor.layers.quantization import QuantizationConfig
from vllm.model_executor.models.utils import extract_layer_index
from vllm.models.deepseek_v41.common.rope import build_deepseek_v4_rope
from vllm.models.deepseek_v41.compressor import DeepseekCompressor
from vllm.platforms import current_platform
from vllm.triton_utils import tl, triton
from vllm.utils.multi_stream_utils import (
execute_in_parallel,
@@ -109,6 +110,16 @@ def _fill_short_context_topk_indices(
)
# Which packed fp8_ds_mla record a V4.1 layer writes. FlashMLA decodes
# DeepSeek's V4.1 record -- all 512 dims (RoPE included) as fp8 e4m3 with one
# UE8M0 scale per 32 dims, 512 data bytes + 16 scale bytes per token, pages
# rounded to the kernel's 512 B TMA stride -- only in its SM100 sparse-decode
# kernels. Every other arch keeps the V4 record: 448 fp8 NoPE + 64 bf16 RoPE
# plus 7 UE8M0 scales of 64 dims and a pad byte (584 B, 576 B pages).
def _use_v41_mxfp8_kv_record() -> bool:
return current_platform.is_device_capability_family(100)
def _resolve_dsv4_kv_cache_dtype(
use_fp8_ds_mla_layout: bool,
kv_cache_dtype: str,
@@ -445,6 +456,9 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
self.kv_cache_dtype, self.kv_cache_torch_dtype = _resolve_dsv4_kv_cache_dtype(
self._uses_fp8_ds_mla_layout(), cache_config.cache_dtype, cache_config
)
self.kv_mxfp8 = _use_v41_mxfp8_kv_record()
self.kv_bytes_per_token = 528 if self.kv_mxfp8 else 584
self.kv_page_alignment = 512 if self.kv_mxfp8 else 576
self.swa_cache_layer = DeepseekV4SWACache(
head_dim=self.head_dim,
@@ -454,6 +468,8 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
cache_config=cache_config,
backend_cls=self.swa_backend_cls,
block_size=32,
packed_bytes_per_token=self.kv_bytes_per_token,
packed_page_alignment=self.kv_page_alignment,
)
# The attention layer itself was already registered with the
@@ -879,6 +895,7 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
self.eps,
swa_metadata.block_size,
False,
self.kv_mxfp8,
)
# Plain-row path: the [num_blocks, block_size, 512] cache stores the KV
@@ -934,9 +951,9 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
# DeepseekV4SWACache.
if not self.is_kv_source:
return None
# fp8_ds_mla is a UE8M0 block-scaled uint8 layout and needs 576B
# alignment; plain bf16 / per-tensor fp8 rows use natural element-size
# pages.
# fp8_ds_mla is a UE8M0 block-scaled uint8 layout whose page rounds up
# to the decode kernel's TMA stride; plain bf16 / per-tensor fp8 rows
# use natural element-size pages.
uses_fp8_ds_mla_layout = self.kv_cache_dtype == "fp8_ds_mla"
return MLAAttentionSpec(
block_size=vllm_config.cache_config.block_size,
@@ -945,12 +962,13 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
dtype=torch.uint8 if uses_fp8_ds_mla_layout else self.kv_cache_torch_dtype,
tokens_per_state=self.compress_ratio,
cache_dtype_str=self.kv_cache_dtype,
alignment=576 if uses_fp8_ds_mla_layout else 512,
alignment=self.kv_page_alignment if uses_fp8_ds_mla_layout else 512,
model_version="deepseek_v4",
kv_quant_mode=get_kv_quant_mode(self.kv_cache_dtype),
# DeepseekV4: 448B NoPE + 128B RoPE + 8B fp8 scale = 584B per token;
# head_size stays semantic (512).
state_content_bytes=584 if uses_fp8_ds_mla_layout else None,
# Packed record width; head_size stays semantic (512).
state_content_bytes=(
self.kv_bytes_per_token if uses_fp8_ds_mla_layout else None
),
)
def _compressed_kv_cache(self) -> torch.Tensor:
@@ -998,8 +1016,15 @@ class DeepseekV4IndexerCache(torch.nn.Module, AttentionLayerBase):
head_size=self.head_dim,
dtype=self.dtype,
tokens_per_state=self.compress_ratio,
# 576B for FlashMLA packing; 512B for FlashInfer sparse (#44577).
alignment=576 if uses_fp8_ds_mla_layout else 512,
# The indexer page shares a physical block with the main KV cache
# (the block stride is the sum of every page in the group), so it
# has to carry the same padding: the packed record's TMA stride,
# else 512B for FlashInfer sparse (#44577).
alignment=(
576
if uses_fp8_ds_mla_layout and not _use_v41_mxfp8_kv_record()
else 512
),
)
def forward(self): ...