[Kernel][DSv4.1] Bump FlashMLA to the 2026.09.30 release and its interleaved V4.1 KV cache (#61371)
Signed-off-by: juntianl <juntianl@inferact.ai> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: Kimi Code <noreply@moonshot.ai> Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai>
This commit is contained in:
co-authored by
Claude Opus 5.5
Kimi Code
Yifan Qiao
parent
33212db2c2
commit
f8460ef236
@@ -427,7 +427,7 @@ def sparse_mqa_logits_paged_decode(
|
||||
q: [rows, 1, H, D] packed Q values (MXFP4 viewed as int8).
|
||||
q_scale: [rows, 1, H] int32 packed UE8M0 Q scales.
|
||||
kv_cache: 4D [num_pages, page_kv, 1, head_bytes] uint8 fused cache
|
||||
view; the page stride must be 512B-aligned.
|
||||
view; the page stride must be 256B-aligned.
|
||||
weights: [rows, H] bf16 per-head weights.
|
||||
context_lens: [rows] int32 per-row (compressed) context lengths.
|
||||
block_table: [rows, P] int32 per-row page table, ``stride(-1) == 1``.
|
||||
@@ -451,8 +451,8 @@ def sparse_mqa_logits_paged_decode(
|
||||
rows = q.shape[0]
|
||||
page_kv = kv_cache.shape[1]
|
||||
assert weights.dtype == torch.bfloat16, weights.dtype
|
||||
assert kv_cache.stride(0) % 512 == 0, (
|
||||
"paged cache page stride must be 512B-aligned: "
|
||||
assert kv_cache.stride(0) % 256 == 0, (
|
||||
"paged cache page stride must be 256B-aligned: "
|
||||
f"shape={tuple(kv_cache.shape)} stride={kv_cache.stride()}"
|
||||
)
|
||||
assert page_kv % sparse_block_kv == 0, (page_kv, sparse_block_kv)
|
||||
|
||||
@@ -52,6 +52,12 @@ class DequantGatherKCacheKernel(
|
||||
data_dim = fp8_dim + bf16_dim * 2
|
||||
scale_bytes = head_dim // group_size
|
||||
block_size = compile_key.block_size
|
||||
# V4.1 stores each token's scales right after its data; V4 puts all of
|
||||
# a block's data rows ahead of its scale rows.
|
||||
interleaved = bf16_dim == 0
|
||||
data_stride = data_dim + scale_bytes if interleaved else data_dim
|
||||
scale_offset = data_dim if interleaved else block_size * data_dim
|
||||
scale_stride = data_stride if interleaved else scale_bytes
|
||||
|
||||
@cute.jit
|
||||
def load_g2s(
|
||||
@@ -274,23 +280,22 @@ class DequantGatherKCacheKernel(
|
||||
offset: Int32,
|
||||
stream: CUstream,
|
||||
):
|
||||
# Split k_cache into k_data and k_scale. Each [block_size, head_bytes]
|
||||
# block is actually a concat of
|
||||
# [block_size, fp8_dim + bf16_dim * 2] and [block_size, scale_bytes].
|
||||
# Split k_cache into k_data [block_size, fp8_dim + bf16_dim * 2] and
|
||||
# k_scale [block_size, scale_bytes] views of each block.
|
||||
k_data = cute.make_tensor(
|
||||
k_cache.iterator,
|
||||
layout=cute.make_layout(
|
||||
(k_cache.shape[0], block_size, data_dim),
|
||||
stride=(k_cache.stride[0], data_dim, 1),
|
||||
stride=(k_cache.stride[0], data_stride, 1),
|
||||
),
|
||||
)
|
||||
# k_data_slice: [num_blocks, block_size, (16, data_dim/16)]
|
||||
# s_kdata_slice: [(4, data_dim/16), num_stages]
|
||||
k_scale = cute.make_tensor(
|
||||
k_cache.iterator + (block_size * data_dim),
|
||||
k_cache.iterator + scale_offset,
|
||||
layout=cute.make_layout(
|
||||
(k_cache.shape[0], block_size, scale_bytes),
|
||||
stride=(k_cache.stride[0], scale_bytes, 1),
|
||||
stride=(k_cache.stride[0], scale_stride, 1),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@@ -122,10 +122,10 @@ def _fill_short_context_topk_indices(
|
||||
|
||||
# Which packed fp8_ds_mla record a V4.1 layer writes. FlashMLA decodes
|
||||
# DeepSeek's V4.1 record -- all 512 dims (RoPE included) as fp8 e4m3 with one
|
||||
# UE8M0 scale per 32 dims, 512 data bytes + 16 scale bytes per token, pages
|
||||
# rounded to the kernel's 512 B TMA stride -- only in its SM100 sparse-decode
|
||||
# kernels. Every other arch keeps the V4 record: 448 fp8 NoPE + 64 bf16 RoPE
|
||||
# plus 7 UE8M0 scales of 64 dims and a pad byte (584 B, 576 B pages).
|
||||
# UE8M0 scale per 32 dims, each token's 512 data bytes followed by its 16 scale
|
||||
# bytes -- only in its SM100 sparse-decode kernels. Every other arch keeps the
|
||||
# V4 record: 448 fp8 NoPE + 64 bf16 RoPE plus 7 UE8M0 scales of 64 dims and a
|
||||
# pad byte (584 B, 576 B pages).
|
||||
def _use_v41_mxfp8_kv_record() -> bool:
|
||||
return current_platform.is_device_capability_family(100)
|
||||
|
||||
@@ -515,8 +515,8 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
|
||||
288 if self.kv_cache_dtype == "nvfp4_ds_mla" else self.swa_bytes_per_token
|
||||
)
|
||||
# One alignment for every page in the block: the block stride is their
|
||||
# sum, and 512 satisfies both TMA strides in play (512 for the V4.1
|
||||
# fp8 record, 256 for NVFP4).
|
||||
# sum. V4 needs its 576 B TMA stride; V4.1's is on the SWA block
|
||||
# stride, and 512 is free (its pages are already 512 B multiples).
|
||||
self.kv_page_alignment = 512 if self.kv_mxfp8 else 576
|
||||
if self.kv_cache_dtype == "nvfp4_ds_mla" and not self.kv_mxfp8:
|
||||
raise ValueError(
|
||||
@@ -562,6 +562,13 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
|
||||
block_size=32,
|
||||
packed_bytes_per_token=self.swa_bytes_per_token,
|
||||
packed_page_alignment=self.kv_page_alignment,
|
||||
# FlashMLA reads the interleaved V4.1 records as rows of the whole
|
||||
# cache, so the packed block has to hold whole rows of both.
|
||||
packed_block_stride_alignment=(
|
||||
math.lcm(self.swa_bytes_per_token, self.compressed_bytes_per_token)
|
||||
if self.kv_mxfp8
|
||||
else None
|
||||
),
|
||||
bounded_replay=swa_bounded_replay,
|
||||
)
|
||||
|
||||
@@ -1222,15 +1229,18 @@ class DeepseekV4IndexerCache(torch.nn.Module, AttentionLayerBase):
|
||||
tokens_per_state=self.compress_ratio,
|
||||
# The indexer page shares a physical block with the main KV cache
|
||||
# (the block stride is the sum of every page in the group), so it
|
||||
# has to carry the same padding: the packed record's TMA stride,
|
||||
# has to carry the same padding: the packed KV pages' alignment,
|
||||
# else 512B for FlashInfer sparse (#44577).
|
||||
alignment=page_alignment,
|
||||
# DeepGEMM's paged sparse MQA logits address pages by the block
|
||||
# stride, which packs every layer's page in this layout, and need
|
||||
# it 512B-aligned; the main KV record read from the same blocks
|
||||
# needs its own TMA stride, so keep both.
|
||||
# DeepGEMM's paged sparse logits need a 256B-aligned block stride;
|
||||
# V4 records keep their TMA stride (V4.1's is on the SWA spec).
|
||||
block_stride_alignment=(
|
||||
math.lcm(512, page_alignment)
|
||||
math.lcm(
|
||||
256,
|
||||
1
|
||||
if uses_fp8_ds_mla_layout and _use_v41_mxfp8_kv_record()
|
||||
else page_alignment,
|
||||
)
|
||||
if self.sparse_logits and not self.rocm_mxfp4
|
||||
else None
|
||||
),
|
||||
|
||||
@@ -36,16 +36,17 @@ from vllm.triton_utils import tl, triton
|
||||
from vllm.utils.import_utils import has_cutedsl
|
||||
from vllm.utils.math_utils import cdiv, next_power_of_2
|
||||
|
||||
# Per-token byte width of the two paged fp8 records. Both put a page's whole
|
||||
# data region ahead of its whole scale region, so ``k_cache.shape[-1]`` -- the
|
||||
# Per-token byte width of the paged records; ``k_cache.shape[-1]`` -- the
|
||||
# width the KV-cache spec publishes -- identifies the record:
|
||||
# V4 (584 B): 448 fp8 e4m3 NoPE + 64 bf16 RoPE, then 7 UE8M0 scales of 64
|
||||
# dims each plus a pad byte.
|
||||
# dims each plus a pad byte. A page puts its whole data region
|
||||
# ahead of its whole scale region.
|
||||
# V4.1 (528 B): all 512 dims as fp8 e4m3 (RoPE included), then 16 UE8M0
|
||||
# scales of 32 dims each. FlashMLA's ``ModelType::V41``.
|
||||
# V4.1 NVFP4 (288 B): 512 e2m1 values packed two per byte (even element in
|
||||
# the low nibble), then 32 e4m3 scales of 16 dims each.
|
||||
# FlashMLA's ``ModelType::V41_FP4``, compressed cache only.
|
||||
# The V4.1 records are interleaved: each token's scales follow its own data.
|
||||
V4_BYTES_PER_TOKEN = 584
|
||||
V41_BYTES_PER_TOKEN = 528
|
||||
V41_QUANT_BLOCK = 32
|
||||
@@ -212,10 +213,8 @@ def _quantize_and_insert_k_mxfp8_kernel(
|
||||
|
||||
# int64: block_idx * block_stride can exceed 2^31 with many KV-cache blocks.
|
||||
cache_block_ptr = k_cache_ptr + block_idx.to(tl.int64) * block_stride
|
||||
token_data_ptr = cache_block_ptr + pos_in_block * head_dim
|
||||
token_scale_ptr = (
|
||||
cache_block_ptr + cache_block_size * head_dim + pos_in_block * scale_dim
|
||||
)
|
||||
token_data_ptr = cache_block_ptr + pos_in_block * (head_dim + scale_dim)
|
||||
token_scale_ptr = token_data_ptr + head_dim
|
||||
|
||||
d = tl.arange(0, head_dim)
|
||||
x = tl.load(k_ptr + pid.to(tl.int64) * head_dim + d).to(tl.float32)
|
||||
@@ -568,10 +567,8 @@ def _dequantize_and_gather_k_mxfp8_kernel(
|
||||
|
||||
# int64: physical_block_idx * block_stride can exceed 2^31.
|
||||
cache_block_ptr = k_cache_ptr + physical_block_idx.to(tl.int64) * block_stride
|
||||
token_data_ptr = cache_block_ptr + pos_in_block * head_dim
|
||||
token_scale_ptr = (
|
||||
cache_block_ptr + cache_block_size * head_dim + pos_in_block * scale_dim
|
||||
)
|
||||
token_data_ptr = cache_block_ptr + pos_in_block * (head_dim + scale_dim)
|
||||
token_scale_ptr = token_data_ptr + head_dim
|
||||
|
||||
x_uint8 = tl.load(token_data_ptr + d)
|
||||
if use_fnuz:
|
||||
@@ -629,10 +626,9 @@ def _dequantize_and_gather_k_nvfp4_kernel(
|
||||
block_table_ptr + batch_idx * max_blocks_per_seq + block_in_seq
|
||||
)
|
||||
page = k_cache_ptr + physical_block_idx.to(tl.int64) * block_stride
|
||||
token_ptr = page + pos_in_block * (packed_bytes + scale_dim)
|
||||
|
||||
packed = tl.load(
|
||||
page + pos_in_block * packed_bytes + tl.arange(0, packed_bytes)
|
||||
)
|
||||
packed = tl.load(token_ptr + tl.arange(0, packed_bytes))
|
||||
# Even element in the low nibble, odd in the high nibble.
|
||||
codes = tl.interleave((packed & 0xF).to(tl.int32), (packed >> 4).to(tl.int32))
|
||||
# e2m1 magnitudes: 0, 0.5, 1, 1.5, 2, 3, 4, 6.
|
||||
@@ -642,12 +638,7 @@ def _dequantize_and_gather_k_nvfp4_kernel(
|
||||
mag = tl.where(mag_code < 2, m * 0.5, (1.0 + m * 0.5) * tl.exp2(e - 1.0))
|
||||
vals = tl.where(codes >= 8, -mag, mag)
|
||||
|
||||
sf = tl.load(
|
||||
page
|
||||
+ cache_block_size * packed_bytes
|
||||
+ pos_in_block * scale_dim
|
||||
+ tl.arange(0, scale_dim)
|
||||
)
|
||||
sf = tl.load(token_ptr + packed_bytes + tl.arange(0, scale_dim))
|
||||
scale = sf.to(tl.float8e4nv, bitcast=True).to(tl.float32)
|
||||
tiles = tl.reshape(vals, (scale_dim, quant_block))
|
||||
dequant = tl.reshape(tiles * tl.reshape(scale, (scale_dim, 1)), (head_dim,))
|
||||
|
||||
@@ -239,7 +239,8 @@ def rope_quant_insert(
|
||||
UE8M0 scale bytes, including one zero padding scale), 528 B for V4.1
|
||||
(512 MXFP8 value bytes covering the RoPE dims too, then 16 UE8M0 scales of
|
||||
32 dims each), or 288 B for V4.1 NVFP4 (256 bytes of e2m1 pairs then 32
|
||||
e4m3 scales of 16 dims each), which only the compressed cache uses.
|
||||
e4m3 scales of 16 dims each), which only the compressed cache uses. The
|
||||
two V4.1 records keep each token's scales right after its own values.
|
||||
``bfloat16`` and ``float8_e4m3fn`` are the plain [448 NoPE | 64 RoPE] rows
|
||||
read by FlashInfer, the latter scaled by the per-tensor ``fp8_scale``.
|
||||
"""
|
||||
@@ -390,8 +391,8 @@ def _rope_quant_insert_mxfp8_kernel(
|
||||
rotated = tl.where(rotated == rotated, rotated, 0.0)
|
||||
|
||||
page = cache + (slot // CACHE_BLOCK).to(tl.int64) * CACHE_STRIDE
|
||||
values = page + (slot % CACHE_BLOCK) * 512
|
||||
scales = page + CACHE_BLOCK * 512 + (slot % CACHE_BLOCK) * 16
|
||||
values = page + (slot % CACHE_BLOCK) * 528
|
||||
scales = values + 512
|
||||
|
||||
quant = tl.reshape(rotated, (16, 32))
|
||||
amax = tl.maximum(tl.max(tl.abs(quant), 1), 1e-4)
|
||||
@@ -453,12 +454,9 @@ def _rope_quant_insert_nvfp4_kernel(
|
||||
lo, hi = tl.split(tl.reshape(tl.reshape(scaled, (512,)), (256, 2)))
|
||||
|
||||
page = cache + (slot // CACHE_BLOCK).to(tl.int64) * CACHE_STRIDE
|
||||
tl.store(
|
||||
page + (slot % CACHE_BLOCK) * 256 + tl.arange(0, 256),
|
||||
_fp32x2_to_fp4x2(lo, hi),
|
||||
)
|
||||
scales = page + CACHE_BLOCK * 256 + (slot % CACHE_BLOCK) * 32
|
||||
tl.store(scales + tl.arange(0, 32), scale.to(tl.uint8, bitcast=True))
|
||||
values = page + (slot % CACHE_BLOCK) * 288
|
||||
tl.store(values + tl.arange(0, 256), _fp32x2_to_fp4x2(lo, hi))
|
||||
tl.store(values + 256 + tl.arange(0, 32), scale.to(tl.uint8, bitcast=True))
|
||||
|
||||
|
||||
@triton.jit
|
||||
|
||||
@@ -948,7 +948,7 @@ def fp8_fp4_paged_sparse_mqa_logits(
|
||||
Args:
|
||||
q: ``(q_values, q_scale)``; q_values is [num_q_tokens, 1, H, D].
|
||||
kv_cache: Fused paged cache [num_pages, page_kv, 1, D' + 4] uint8,
|
||||
page stride 512B-aligned.
|
||||
page stride 256B-aligned.
|
||||
weights: [num_q_tokens, H] ``torch.bfloat16``.
|
||||
metadata: From `get_paged_sparse_mqa_logits_metadata`.
|
||||
num_max_sparse_blocks: Candidate blocks per row.
|
||||
|
||||
@@ -103,10 +103,11 @@ For DeepSeek V4.1, each token's KV cache is 528 Bytes, structured as:
|
||||
- **Last 16 bytes:** Scale factors, containing 16 `ue8m0` values, one per 32
|
||||
consecutive `float8_e4m3` values (i.e. MXFP8).
|
||||
|
||||
The V4 and V4.1 records are not laid out token-by-token within a page: a page
|
||||
holds all its data rows first and all its scale rows after, so a page is
|
||||
`block_size * bytes_per_token` bytes rounded up to the decode kernel's TMA
|
||||
stride (576 B for V4, 512 B for V4.1).
|
||||
The V4 record is not laid out token-by-token within a page: a page holds all
|
||||
its data rows first and all its scale rows after, so a page is
|
||||
`block_size * bytes_per_token` bytes rounded up to the decode kernel's 576 B
|
||||
TMA stride. The V4.1 records are interleaved: each token's scales directly
|
||||
follow its own data, and the decode kernel's TMA stride is the record width.
|
||||
|
||||
In the "nvfp4_ds_mla" format (SM100 only, DeepSeek V3.2 geometry), each
|
||||
token's KV cache is 352 Bytes, structured as:
|
||||
|
||||
@@ -85,6 +85,7 @@ class DeepseekV4SWACache(torch.nn.Module, AttentionLayerBase):
|
||||
block_size: int = 64,
|
||||
packed_bytes_per_token: int = 584,
|
||||
packed_page_alignment: int = 576,
|
||||
packed_block_stride_alignment: int | None = None,
|
||||
bounded_replay: bool = False,
|
||||
):
|
||||
super().__init__()
|
||||
@@ -99,6 +100,7 @@ class DeepseekV4SWACache(torch.nn.Module, AttentionLayerBase):
|
||||
self.dtype = dtype
|
||||
self.packed_bytes_per_token = packed_bytes_per_token
|
||||
self.packed_page_alignment = packed_page_alignment
|
||||
self.packed_block_stride_alignment = packed_block_stride_alignment
|
||||
compilation_config = get_current_vllm_config().compilation_config
|
||||
if prefix in compilation_config.static_forward_context:
|
||||
raise ValueError(f"Duplicate layer name: {prefix}")
|
||||
@@ -135,8 +137,11 @@ class DeepseekV4SWACache(torch.nn.Module, AttentionLayerBase):
|
||||
state_content_bytes=(
|
||||
self.packed_bytes_per_token if uses_fp8_ds_mla_layout else None
|
||||
),
|
||||
# FlashMLA packing stride; 512B for FlashInfer sparse (#44577).
|
||||
# Packed page alignment; 512B for FlashInfer sparse (#44577).
|
||||
alignment=self.packed_page_alignment if uses_fp8_ds_mla_layout else 512,
|
||||
block_stride_alignment=(
|
||||
self.packed_block_stride_alignment if uses_fp8_ds_mla_layout else None
|
||||
),
|
||||
model_version="deepseek_v4",
|
||||
kv_quant_mode=get_kv_quant_mode(self.cache_config.cache_dtype),
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user