[Kernel][DSv4.1] Bump FlashMLA to the 2026.09.30 release and its interleaved V4.1 KV cache (#61371)

Signed-off-by: juntianl <juntianl@inferact.ai>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: Kimi Code <noreply@moonshot.ai>
Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai>
This commit is contained in:
Juntian Liu
2026-10-15 13:58:25 -07:00
committed by GitHub
co-authored by Claude Opus 5.5 Kimi Code Yifan Qiao
parent 33212db2c2
commit f8460ef236
13 changed files with 262 additions and 165 deletions
@@ -427,7 +427,7 @@ def sparse_mqa_logits_paged_decode(
q: [rows, 1, H, D] packed Q values (MXFP4 viewed as int8).
q_scale: [rows, 1, H] int32 packed UE8M0 Q scales.
kv_cache: 4D [num_pages, page_kv, 1, head_bytes] uint8 fused cache
view; the page stride must be 512B-aligned.
view; the page stride must be 256B-aligned.
weights: [rows, H] bf16 per-head weights.
context_lens: [rows] int32 per-row (compressed) context lengths.
block_table: [rows, P] int32 per-row page table, ``stride(-1) == 1``.
@@ -451,8 +451,8 @@ def sparse_mqa_logits_paged_decode(
rows = q.shape[0]
page_kv = kv_cache.shape[1]
assert weights.dtype == torch.bfloat16, weights.dtype
assert kv_cache.stride(0) % 512 == 0, (
"paged cache page stride must be 512B-aligned: "
assert kv_cache.stride(0) % 256 == 0, (
"paged cache page stride must be 256B-aligned: "
f"shape={tuple(kv_cache.shape)} stride={kv_cache.stride()}"
)
assert page_kv % sparse_block_kv == 0, (page_kv, sparse_block_kv)
@@ -52,6 +52,12 @@ class DequantGatherKCacheKernel(
data_dim = fp8_dim + bf16_dim * 2
scale_bytes = head_dim // group_size
block_size = compile_key.block_size
# V4.1 stores each token's scales right after its data; V4 puts all of
# a block's data rows ahead of its scale rows.
interleaved = bf16_dim == 0
data_stride = data_dim + scale_bytes if interleaved else data_dim
scale_offset = data_dim if interleaved else block_size * data_dim
scale_stride = data_stride if interleaved else scale_bytes
@cute.jit
def load_g2s(
@@ -274,23 +280,22 @@ class DequantGatherKCacheKernel(
offset: Int32,
stream: CUstream,
):
# Split k_cache into k_data and k_scale. Each [block_size, head_bytes]
# block is actually a concat of
# [block_size, fp8_dim + bf16_dim * 2] and [block_size, scale_bytes].
# Split k_cache into k_data [block_size, fp8_dim + bf16_dim * 2] and
# k_scale [block_size, scale_bytes] views of each block.
k_data = cute.make_tensor(
k_cache.iterator,
layout=cute.make_layout(
(k_cache.shape[0], block_size, data_dim),
stride=(k_cache.stride[0], data_dim, 1),
stride=(k_cache.stride[0], data_stride, 1),
),
)
# k_data_slice: [num_blocks, block_size, (16, data_dim/16)]
# s_kdata_slice: [(4, data_dim/16), num_stages]
k_scale = cute.make_tensor(
k_cache.iterator + (block_size * data_dim),
k_cache.iterator + scale_offset,
layout=cute.make_layout(
(k_cache.shape[0], block_size, scale_bytes),
stride=(k_cache.stride[0], scale_bytes, 1),
stride=(k_cache.stride[0], scale_stride, 1),
),
)
+22 -12
View File
@@ -122,10 +122,10 @@ def _fill_short_context_topk_indices(
# Which packed fp8_ds_mla record a V4.1 layer writes. FlashMLA decodes
# DeepSeek's V4.1 record -- all 512 dims (RoPE included) as fp8 e4m3 with one
# UE8M0 scale per 32 dims, 512 data bytes + 16 scale bytes per token, pages
# rounded to the kernel's 512 B TMA stride -- only in its SM100 sparse-decode
# kernels. Every other arch keeps the V4 record: 448 fp8 NoPE + 64 bf16 RoPE
# plus 7 UE8M0 scales of 64 dims and a pad byte (584 B, 576 B pages).
# UE8M0 scale per 32 dims, each token's 512 data bytes followed by its 16 scale
# bytes -- only in its SM100 sparse-decode kernels. Every other arch keeps the
# V4 record: 448 fp8 NoPE + 64 bf16 RoPE plus 7 UE8M0 scales of 64 dims and a
# pad byte (584 B, 576 B pages).
def _use_v41_mxfp8_kv_record() -> bool:
return current_platform.is_device_capability_family(100)
@@ -515,8 +515,8 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
288 if self.kv_cache_dtype == "nvfp4_ds_mla" else self.swa_bytes_per_token
)
# One alignment for every page in the block: the block stride is their
# sum, and 512 satisfies both TMA strides in play (512 for the V4.1
# fp8 record, 256 for NVFP4).
# sum. V4 needs its 576 B TMA stride; V4.1's is on the SWA block
# stride, and 512 is free (its pages are already 512 B multiples).
self.kv_page_alignment = 512 if self.kv_mxfp8 else 576
if self.kv_cache_dtype == "nvfp4_ds_mla" and not self.kv_mxfp8:
raise ValueError(
@@ -562,6 +562,13 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
block_size=32,
packed_bytes_per_token=self.swa_bytes_per_token,
packed_page_alignment=self.kv_page_alignment,
# FlashMLA reads the interleaved V4.1 records as rows of the whole
# cache, so the packed block has to hold whole rows of both.
packed_block_stride_alignment=(
math.lcm(self.swa_bytes_per_token, self.compressed_bytes_per_token)
if self.kv_mxfp8
else None
),
bounded_replay=swa_bounded_replay,
)
@@ -1222,15 +1229,18 @@ class DeepseekV4IndexerCache(torch.nn.Module, AttentionLayerBase):
tokens_per_state=self.compress_ratio,
# The indexer page shares a physical block with the main KV cache
# (the block stride is the sum of every page in the group), so it
# has to carry the same padding: the packed record's TMA stride,
# has to carry the same padding: the packed KV pages' alignment,
# else 512B for FlashInfer sparse (#44577).
alignment=page_alignment,
# DeepGEMM's paged sparse MQA logits address pages by the block
# stride, which packs every layer's page in this layout, and need
# it 512B-aligned; the main KV record read from the same blocks
# needs its own TMA stride, so keep both.
# DeepGEMM's paged sparse logits need a 256B-aligned block stride;
# V4 records keep their TMA stride (V4.1's is on the SWA spec).
block_stride_alignment=(
math.lcm(512, page_alignment)
math.lcm(
256,
1
if uses_fp8_ds_mla_layout and _use_v41_mxfp8_kv_record()
else page_alignment,
)
if self.sparse_logits and not self.rocm_mxfp4
else None
),
@@ -36,16 +36,17 @@ from vllm.triton_utils import tl, triton
from vllm.utils.import_utils import has_cutedsl
from vllm.utils.math_utils import cdiv, next_power_of_2
# Per-token byte width of the two paged fp8 records. Both put a page's whole
# data region ahead of its whole scale region, so ``k_cache.shape[-1]`` -- the
# Per-token byte width of the paged records; ``k_cache.shape[-1]`` -- the
# width the KV-cache spec publishes -- identifies the record:
# V4 (584 B): 448 fp8 e4m3 NoPE + 64 bf16 RoPE, then 7 UE8M0 scales of 64
# dims each plus a pad byte.
# dims each plus a pad byte. A page puts its whole data region
# ahead of its whole scale region.
# V4.1 (528 B): all 512 dims as fp8 e4m3 (RoPE included), then 16 UE8M0
# scales of 32 dims each. FlashMLA's ``ModelType::V41``.
# V4.1 NVFP4 (288 B): 512 e2m1 values packed two per byte (even element in
# the low nibble), then 32 e4m3 scales of 16 dims each.
# FlashMLA's ``ModelType::V41_FP4``, compressed cache only.
# The V4.1 records are interleaved: each token's scales follow its own data.
V4_BYTES_PER_TOKEN = 584
V41_BYTES_PER_TOKEN = 528
V41_QUANT_BLOCK = 32
@@ -212,10 +213,8 @@ def _quantize_and_insert_k_mxfp8_kernel(
# int64: block_idx * block_stride can exceed 2^31 with many KV-cache blocks.
cache_block_ptr = k_cache_ptr + block_idx.to(tl.int64) * block_stride
token_data_ptr = cache_block_ptr + pos_in_block * head_dim
token_scale_ptr = (
cache_block_ptr + cache_block_size * head_dim + pos_in_block * scale_dim
)
token_data_ptr = cache_block_ptr + pos_in_block * (head_dim + scale_dim)
token_scale_ptr = token_data_ptr + head_dim
d = tl.arange(0, head_dim)
x = tl.load(k_ptr + pid.to(tl.int64) * head_dim + d).to(tl.float32)
@@ -568,10 +567,8 @@ def _dequantize_and_gather_k_mxfp8_kernel(
# int64: physical_block_idx * block_stride can exceed 2^31.
cache_block_ptr = k_cache_ptr + physical_block_idx.to(tl.int64) * block_stride
token_data_ptr = cache_block_ptr + pos_in_block * head_dim
token_scale_ptr = (
cache_block_ptr + cache_block_size * head_dim + pos_in_block * scale_dim
)
token_data_ptr = cache_block_ptr + pos_in_block * (head_dim + scale_dim)
token_scale_ptr = token_data_ptr + head_dim
x_uint8 = tl.load(token_data_ptr + d)
if use_fnuz:
@@ -629,10 +626,9 @@ def _dequantize_and_gather_k_nvfp4_kernel(
block_table_ptr + batch_idx * max_blocks_per_seq + block_in_seq
)
page = k_cache_ptr + physical_block_idx.to(tl.int64) * block_stride
token_ptr = page + pos_in_block * (packed_bytes + scale_dim)
packed = tl.load(
page + pos_in_block * packed_bytes + tl.arange(0, packed_bytes)
)
packed = tl.load(token_ptr + tl.arange(0, packed_bytes))
# Even element in the low nibble, odd in the high nibble.
codes = tl.interleave((packed & 0xF).to(tl.int32), (packed >> 4).to(tl.int32))
# e2m1 magnitudes: 0, 0.5, 1, 1.5, 2, 3, 4, 6.
@@ -642,12 +638,7 @@ def _dequantize_and_gather_k_nvfp4_kernel(
mag = tl.where(mag_code < 2, m * 0.5, (1.0 + m * 0.5) * tl.exp2(e - 1.0))
vals = tl.where(codes >= 8, -mag, mag)
sf = tl.load(
page
+ cache_block_size * packed_bytes
+ pos_in_block * scale_dim
+ tl.arange(0, scale_dim)
)
sf = tl.load(token_ptr + packed_bytes + tl.arange(0, scale_dim))
scale = sf.to(tl.float8e4nv, bitcast=True).to(tl.float32)
tiles = tl.reshape(vals, (scale_dim, quant_block))
dequant = tl.reshape(tiles * tl.reshape(scale, (scale_dim, 1)), (head_dim,))
@@ -239,7 +239,8 @@ def rope_quant_insert(
UE8M0 scale bytes, including one zero padding scale), 528 B for V4.1
(512 MXFP8 value bytes covering the RoPE dims too, then 16 UE8M0 scales of
32 dims each), or 288 B for V4.1 NVFP4 (256 bytes of e2m1 pairs then 32
e4m3 scales of 16 dims each), which only the compressed cache uses.
e4m3 scales of 16 dims each), which only the compressed cache uses. The
two V4.1 records keep each token's scales right after its own values.
``bfloat16`` and ``float8_e4m3fn`` are the plain [448 NoPE | 64 RoPE] rows
read by FlashInfer, the latter scaled by the per-tensor ``fp8_scale``.
"""
@@ -390,8 +391,8 @@ def _rope_quant_insert_mxfp8_kernel(
rotated = tl.where(rotated == rotated, rotated, 0.0)
page = cache + (slot // CACHE_BLOCK).to(tl.int64) * CACHE_STRIDE
values = page + (slot % CACHE_BLOCK) * 512
scales = page + CACHE_BLOCK * 512 + (slot % CACHE_BLOCK) * 16
values = page + (slot % CACHE_BLOCK) * 528
scales = values + 512
quant = tl.reshape(rotated, (16, 32))
amax = tl.maximum(tl.max(tl.abs(quant), 1), 1e-4)
@@ -453,12 +454,9 @@ def _rope_quant_insert_nvfp4_kernel(
lo, hi = tl.split(tl.reshape(tl.reshape(scaled, (512,)), (256, 2)))
page = cache + (slot // CACHE_BLOCK).to(tl.int64) * CACHE_STRIDE
tl.store(
page + (slot % CACHE_BLOCK) * 256 + tl.arange(0, 256),
_fp32x2_to_fp4x2(lo, hi),
)
scales = page + CACHE_BLOCK * 256 + (slot % CACHE_BLOCK) * 32
tl.store(scales + tl.arange(0, 32), scale.to(tl.uint8, bitcast=True))
values = page + (slot % CACHE_BLOCK) * 288
tl.store(values + tl.arange(0, 256), _fp32x2_to_fp4x2(lo, hi))
tl.store(values + 256 + tl.arange(0, 32), scale.to(tl.uint8, bitcast=True))
@triton.jit
+1 -1
View File
@@ -948,7 +948,7 @@ def fp8_fp4_paged_sparse_mqa_logits(
Args:
q: ``(q_values, q_scale)``; q_values is [num_q_tokens, 1, H, D].
kv_cache: Fused paged cache [num_pages, page_kv, 1, D' + 4] uint8,
page stride 512B-aligned.
page stride 256B-aligned.
weights: [num_q_tokens, H] ``torch.bfloat16``.
metadata: From `get_paged_sparse_mqa_logits_metadata`.
num_max_sparse_blocks: Candidate blocks per row.
@@ -103,10 +103,11 @@ For DeepSeek V4.1, each token's KV cache is 528 Bytes, structured as:
- **Last 16 bytes:** Scale factors, containing 16 `ue8m0` values, one per 32
consecutive `float8_e4m3` values (i.e. MXFP8).
The V4 and V4.1 records are not laid out token-by-token within a page: a page
holds all its data rows first and all its scale rows after, so a page is
`block_size * bytes_per_token` bytes rounded up to the decode kernel's TMA
stride (576 B for V4, 512 B for V4.1).
The V4 record is not laid out token-by-token within a page: a page holds all
its data rows first and all its scale rows after, so a page is
`block_size * bytes_per_token` bytes rounded up to the decode kernel's 576 B
TMA stride. The V4.1 records are interleaved: each token's scales directly
follow its own data, and the decode kernel's TMA stride is the record width.
In the "nvfp4_ds_mla" format (SM100 only, DeepSeek V3.2 geometry), each
token's KV cache is 352 Bytes, structured as:
+6 -1
View File
@@ -85,6 +85,7 @@ class DeepseekV4SWACache(torch.nn.Module, AttentionLayerBase):
block_size: int = 64,
packed_bytes_per_token: int = 584,
packed_page_alignment: int = 576,
packed_block_stride_alignment: int | None = None,
bounded_replay: bool = False,
):
super().__init__()
@@ -99,6 +100,7 @@ class DeepseekV4SWACache(torch.nn.Module, AttentionLayerBase):
self.dtype = dtype
self.packed_bytes_per_token = packed_bytes_per_token
self.packed_page_alignment = packed_page_alignment
self.packed_block_stride_alignment = packed_block_stride_alignment
compilation_config = get_current_vllm_config().compilation_config
if prefix in compilation_config.static_forward_context:
raise ValueError(f"Duplicate layer name: {prefix}")
@@ -135,8 +137,11 @@ class DeepseekV4SWACache(torch.nn.Module, AttentionLayerBase):
state_content_bytes=(
self.packed_bytes_per_token if uses_fp8_ds_mla_layout else None
),
# FlashMLA packing stride; 512B for FlashInfer sparse (#44577).
# Packed page alignment; 512B for FlashInfer sparse (#44577).
alignment=self.packed_page_alignment if uses_fp8_ds_mla_layout else 512,
block_stride_alignment=(
self.packed_block_stride_alignment if uses_fp8_ds_mla_layout else None
),
model_version="deepseek_v4",
kv_quant_mode=get_kv_quant_mode(self.cache_config.cache_dtype),
)