[Model][DSv4.1] FlashMLA mega attention and the NVFP4 compressed KV cache (#56935)
Signed-off-by: Yongye Zhu <yongye@inferact.ai> Signed-off-by: 云挚 <ningyunxiao.nyx@antgroup.com> Signed-off-by: Yongye Zhu <zyy1102000@gmail.com> Co-authored-by: Yongye Zhu <yongye@inferact.ai> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com> Co-authored-by: Yunxiao Ning <73625538+foraxe@users.noreply.github.com> Co-authored-by: AI Assistant <noreply@openai.com>
This commit is contained in:
co-authored by
Yongye Zhu
Claude Opus 5
Yunxiao Ning
AI Assistant
parent
ceb87de065
commit
d6a1677d55
@@ -331,6 +331,19 @@ def test_v41_fused_save_compress_and_insert(
|
||||
torch.testing.assert_close(cache_backing, expected_cache, rtol=0, atol=0)
|
||||
|
||||
|
||||
def _rotate_rope_tail(
|
||||
latent_row: torch.Tensor, cos_sin_row: torch.Tensor
|
||||
) -> torch.Tensor:
|
||||
"""GPT-J RoPE of one latent row's last 64 dims, in fp32."""
|
||||
row = latent_row.float()
|
||||
c, s = cos_sin_row.chunk(2)
|
||||
rotated = row.clone()
|
||||
even, odd = row[448::2], row[449::2]
|
||||
rotated[448::2] = even * c - odd * s
|
||||
rotated[449::2] = odd * c + even * s
|
||||
return rotated
|
||||
|
||||
|
||||
def _mxfp8_record_reference(
|
||||
latent_row: torch.Tensor, cos_sin_row: torch.Tensor
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
@@ -339,12 +352,7 @@ def _mxfp8_record_reference(
|
||||
Mirrors FlashMLA's ``KVCacheLayout.V41_FP8Sparse`` quantizer: rotate the
|
||||
RoPE tail first, then scale every 32-dim tile, RoPE tiles included.
|
||||
"""
|
||||
row = latent_row.float()
|
||||
c, s = cos_sin_row.chunk(2)
|
||||
rotated = row.clone()
|
||||
even, odd = row[448::2], row[449::2]
|
||||
rotated[448::2] = even * c - odd * s
|
||||
rotated[449::2] = odd * c + even * s
|
||||
rotated = _rotate_rope_tail(latent_row, cos_sin_row)
|
||||
quantized, scales = _ue8m0_reference(rotated, 32, 448.0)
|
||||
return quantized.view(torch.uint8), (scales.log2() + 127).to(torch.uint8)
|
||||
|
||||
@@ -461,6 +469,125 @@ def test_v41_mxfp8_cache_round_trip():
|
||||
assert (error <= tolerance.repeat_interleave(32, dim=-1)).all()
|
||||
|
||||
|
||||
# e2m1 magnitudes indexed by the low 3 bits of a code; bit 3 is the sign.
|
||||
_E2M1_MAGNITUDES = [0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0]
|
||||
|
||||
|
||||
def _decode_nvfp4_row(packed: torch.Tensor, scales: torch.Tensor) -> torch.Tensor:
|
||||
"""Unpack one 256-byte e2m1 row and apply its 32 e4m3 tile scales."""
|
||||
mags = torch.tensor(_E2M1_MAGNITUDES, device=packed.device)
|
||||
codes = torch.empty(512, dtype=torch.uint8, device=packed.device)
|
||||
codes[0::2] = packed & 0xF # even element in the low nibble
|
||||
codes[1::2] = packed >> 4
|
||||
vals = mags[(codes & 7).long()] * torch.where(codes >= 8, -1.0, 1.0)
|
||||
return (
|
||||
vals.view(32, 16) * scales.view(torch.float8_e4m3fn).float().view(32, 1)
|
||||
).flatten()
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA only")
|
||||
@pytest.mark.parametrize("compress_ratio", [1, 2])
|
||||
def test_v41_rope_insert_nvfp4_record(compress_ratio: int):
|
||||
"""The 288-byte V4.1 NVFP4 record: 256 B of e2m1 pairs then 32 e4m3 scales.
|
||||
|
||||
Scales are byte-exact against the reference (``amax / 6`` clamped to the
|
||||
e4m3 range); values are checked as round-to-nearest on the e2m1 grid, whose
|
||||
coarsest step is 2 between 4 and 6, so half a step is ``1.0 * scale``.
|
||||
"""
|
||||
from vllm.models.deepseek_v41.common.ops.fused_compress_quant_cache import (
|
||||
rope_quant_insert,
|
||||
)
|
||||
|
||||
torch.manual_seed(13)
|
||||
device = "cuda"
|
||||
cache_block = 64
|
||||
cache_stride = math.ceil(cache_block * 288 / 512) * 512
|
||||
num_tokens = 12
|
||||
|
||||
positions = torch.arange(num_tokens, dtype=torch.int64, device=device)
|
||||
latent = torch.randn(num_tokens, 512, dtype=torch.bfloat16, device=device)
|
||||
angles = torch.randn(64, 32, device=device)
|
||||
cos_sin = torch.cat((angles.cos(), angles.sin()), dim=-1)
|
||||
|
||||
cache_backing = torch.full((2, cache_stride), 165, dtype=torch.uint8, device=device)
|
||||
cache = cache_backing.as_strided((2, cache_block, 288), (cache_stride, 288, 1))
|
||||
slots = torch.arange(num_tokens, dtype=torch.int64, device=device)
|
||||
slots[3] = -1 # covers the negative-slot skip
|
||||
|
||||
rope_quant_insert(latent, positions, cos_sin, cache, slots, compress_ratio)
|
||||
|
||||
untouched = torch.full_like(cache_backing, 165)
|
||||
for t in range(num_tokens):
|
||||
slot = slots[t].item()
|
||||
pos = positions[t].item()
|
||||
if slot < 0 or (pos + 1) % compress_ratio:
|
||||
continue
|
||||
page, row = divmod(slot, cache_block)
|
||||
cs = cos_sin[pos // compress_ratio * compress_ratio]
|
||||
rotated = _rotate_rope_tail(latent[t], cs)
|
||||
amax = rotated.view(32, 16).abs().amax(-1)
|
||||
scale = (amax / 6.0).clamp(2.0**-9, 448.0).to(torch.float8_e4m3fn)
|
||||
|
||||
got_scales = cache_backing[page, cache_block * 256 + row * 32 :][:32]
|
||||
torch.testing.assert_close(got_scales, scale.view(torch.uint8), rtol=0, atol=0)
|
||||
decoded = _decode_nvfp4_row(
|
||||
cache_backing[page, row * 256 : (row + 1) * 256], got_scales
|
||||
)
|
||||
step = scale.float().repeat_interleave(16)
|
||||
assert ((decoded - rotated).abs() <= step).all()
|
||||
untouched[page, row * 256 : (row + 1) * 256] = cache_backing[
|
||||
page, row * 256 : (row + 1) * 256
|
||||
]
|
||||
untouched[page, cache_block * 256 + row * 32 :][:32] = got_scales
|
||||
# Rows the kernel must not have touched, page padding included.
|
||||
torch.testing.assert_close(cache_backing, untouched, rtol=0, atol=0)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA only")
|
||||
def test_v41_nvfp4_gather_matches_insert():
|
||||
"""The NVFP4 gather dequantizes exactly what the insert kernel wrote."""
|
||||
from vllm.models.deepseek_v41.common.ops import dequantize_and_gather_k_cache
|
||||
from vllm.models.deepseek_v41.common.ops.fused_compress_quant_cache import (
|
||||
rope_quant_insert,
|
||||
)
|
||||
|
||||
torch.manual_seed(17)
|
||||
device = "cuda"
|
||||
block_size = 64
|
||||
num_tokens = 70
|
||||
num_blocks = 4
|
||||
page_bytes = math.ceil(block_size * 288 / 512) * 512
|
||||
|
||||
positions = torch.arange(num_tokens, dtype=torch.int64, device=device)
|
||||
latent = torch.randn(num_tokens, 512, dtype=torch.bfloat16, device=device)
|
||||
angles = torch.randn(128, 32, device=device)
|
||||
cos_sin = torch.cat((angles.cos(), angles.sin()), dim=-1)
|
||||
backing = torch.zeros(num_blocks, page_bytes, dtype=torch.uint8, device=device)
|
||||
cache = backing.as_strided((num_blocks, block_size, 288), (page_bytes, 288, 1))
|
||||
slots = torch.arange(num_tokens, dtype=torch.int64, device=device)
|
||||
rope_quant_insert(latent, positions, cos_sin, cache, slots, 1)
|
||||
|
||||
out = torch.zeros(1, num_tokens, 512, dtype=torch.bfloat16, device=device)
|
||||
dequantize_and_gather_k_cache(
|
||||
out,
|
||||
cache,
|
||||
seq_lens=torch.tensor([num_tokens], dtype=torch.int32, device=device),
|
||||
gather_lens=None,
|
||||
block_table=torch.arange(num_blocks, dtype=torch.int32, device=device).view(
|
||||
1, -1
|
||||
),
|
||||
block_size=block_size,
|
||||
offset=0,
|
||||
)
|
||||
for t in (0, 1, block_size, num_tokens - 1):
|
||||
page, row = divmod(t, block_size)
|
||||
expected = _decode_nvfp4_row(
|
||||
backing[page, row * 256 : (row + 1) * 256],
|
||||
backing[page, block_size * 256 + row * 32 :][:32],
|
||||
)
|
||||
torch.testing.assert_close(out[0, t].float(), expected, rtol=0, atol=0)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA only")
|
||||
@pytest.mark.parametrize("compress_ratio", [1, 2])
|
||||
@pytest.mark.parametrize("store_fp8", [False, True])
|
||||
|
||||
@@ -0,0 +1,71 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Layout plumbing for DeepSeek V4.1 mega attention.
|
||||
|
||||
The kernel's Q and O layouts are produced by permuting ``wq_b`` rows and
|
||||
``wo_a`` columns once at load, so what has to hold is that the permuted GEMMs
|
||||
agree with a reference that permutes the activation instead.
|
||||
"""
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.models.deepseek_v41.common.ops.fused_layout import (
|
||||
WV_GROUP_SIZE,
|
||||
o_fused_chunk_permutation,
|
||||
o_fused_permutation,
|
||||
permute_q_to_fused,
|
||||
permute_wo_a_,
|
||||
permute_wq_b_,
|
||||
q_fused_permutation,
|
||||
)
|
||||
|
||||
HEAD_DIM = 512
|
||||
|
||||
|
||||
def test_permuted_wq_b_gemm_matches_permuted_activation():
|
||||
"""Permuting wq_b's rows makes the Q GEMM emit the fused layout directly."""
|
||||
torch.manual_seed(0)
|
||||
num_heads, q_lora_rank, num_tokens = 16, 64, 5
|
||||
weight = torch.randn(num_heads * HEAD_DIM, q_lora_rank)
|
||||
# One scale row per weight row, as an MXFP8 wq_b shard carries.
|
||||
scale = torch.randint(
|
||||
0, 256, (num_heads * HEAD_DIM, q_lora_rank // 32), dtype=torch.uint8
|
||||
)
|
||||
qr = torch.randn(num_tokens, q_lora_rank)
|
||||
orig_weight, orig_scale = weight.clone(), scale.clone()
|
||||
|
||||
standard = (qr @ weight.T).view(num_tokens, num_heads, HEAD_DIM)
|
||||
expected = permute_q_to_fused(standard)
|
||||
|
||||
permute_wq_b_(weight, scale, num_heads)
|
||||
fused = (qr @ weight.T).view(num_tokens, num_heads, HEAD_DIM)
|
||||
torch.testing.assert_close(fused, expected)
|
||||
# The scale has to follow its row, or dequant reads another row's scale.
|
||||
perm = q_fused_permutation(num_heads, HEAD_DIM)
|
||||
torch.testing.assert_close(weight, orig_weight[perm])
|
||||
torch.testing.assert_close(scale, orig_scale[perm])
|
||||
|
||||
|
||||
def test_permuted_wo_a_consumes_fused_output():
|
||||
"""Permuting wo_a's input columns lets it read the kernel's O layout."""
|
||||
torch.manual_seed(1)
|
||||
out_features, num_tokens = 32, 4
|
||||
in_features = WV_GROUP_SIZE * HEAD_DIM
|
||||
weight = torch.randn(out_features, in_features)
|
||||
scale = torch.randint(0, 256, (out_features, in_features // 32), dtype=torch.uint8)
|
||||
standard_o = torch.randn(num_tokens, in_features)
|
||||
orig_weight, orig_scale = weight.clone(), scale.clone()
|
||||
|
||||
expected = standard_o @ weight.T
|
||||
# The kernel emits the same values in the fused chunk order.
|
||||
perm = o_fused_permutation(WV_GROUP_SIZE, HEAD_DIM)
|
||||
fused_o = standard_o[:, perm]
|
||||
|
||||
permute_wo_a_(weight, scale, WV_GROUP_SIZE)
|
||||
torch.testing.assert_close(weight, orig_weight[:, perm])
|
||||
torch.testing.assert_close(
|
||||
scale, orig_scale[:, o_fused_chunk_permutation(WV_GROUP_SIZE, HEAD_DIM)]
|
||||
)
|
||||
# Unlike wq_b, this permutation sits inside the 4096-term reduction, so the
|
||||
# summation order changes and the result differs in the last few ulps.
|
||||
torch.testing.assert_close(fused_o @ weight.T, expected, rtol=1e-3, atol=1e-3)
|
||||
@@ -522,6 +522,189 @@ def test_kv_path_mxfp8_record_matches_reference(num_tokens: int, block_size: int
|
||||
)
|
||||
|
||||
|
||||
# ── Test 2d: mega-attention Q layout (is_q_interleaved) ──────────────────────
|
||||
|
||||
Q_CHUNK = 16
|
||||
NUM_Q_CHUNKS = HEAD_DIM // Q_CHUNK # 32
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_tokens", [1, 17, 2048])
|
||||
@pytest.mark.parametrize("n_heads,q_head_padded", [(16, 64), (64, 128), (64, 0)])
|
||||
def test_q_interleaved_pads_without_rope(
|
||||
num_tokens: int, n_heads: int, q_head_padded: int
|
||||
):
|
||||
"""``is_q_interleaved`` with no norm or RoPE only zero-pads Q.
|
||||
|
||||
The mega-attention kernel norms and rotates Q itself, so every live
|
||||
element must survive bit-exact while the padding heads -- which interleave
|
||||
with the live ones inside each of the 32 head-dim chunks -- come back
|
||||
zero. ``q_head_padded=0`` drops the Q pass entirely and returns nothing,
|
||||
for a shard already at the kernel's head count.
|
||||
"""
|
||||
torch.manual_seed(5)
|
||||
device = "cuda"
|
||||
dtype = torch.bfloat16
|
||||
block_size = 64
|
||||
|
||||
q = torch.randn(num_tokens, n_heads, HEAD_DIM, dtype=dtype, device=device)
|
||||
kv = torch.randn(num_tokens, HEAD_DIM, dtype=dtype, device=device)
|
||||
positions = torch.arange(num_tokens, dtype=torch.int64, device=device)
|
||||
cos_sin_cache = make_cos_sin_cache(4096, ROPE_DIM, torch.float32, device)
|
||||
num_blocks = (num_tokens + block_size - 1) // block_size + 1
|
||||
slot_mapping = torch.arange(num_tokens, dtype=torch.int64, device=device)
|
||||
k_cache = torch.zeros(
|
||||
num_blocks, block_size * V41_HEAD_BYTES, dtype=torch.uint8, device=device
|
||||
)
|
||||
|
||||
q_out = torch.ops._C.fused_deepseek_v4_qnorm_rope_kv_rope_quant_insert(
|
||||
q,
|
||||
kv,
|
||||
k_cache,
|
||||
slot_mapping,
|
||||
positions,
|
||||
cos_sin_cache,
|
||||
q_head_padded,
|
||||
1e-6,
|
||||
block_size,
|
||||
False, # apply_q_norm
|
||||
True, # kv_mxfp8
|
||||
False, # apply_q_rope
|
||||
True, # is_q_interleaved
|
||||
)
|
||||
|
||||
# The KV half is unaffected by the Q mode: it must still be inserted.
|
||||
assert k_cache.any()
|
||||
|
||||
if q_head_padded == 0:
|
||||
assert q_out.numel() == 0
|
||||
return
|
||||
assert q_out.shape == (num_tokens, q_head_padded, HEAD_DIM)
|
||||
got = q_out.view(num_tokens, NUM_Q_CHUNKS, q_head_padded, Q_CHUNK)
|
||||
want = q.view(num_tokens, NUM_Q_CHUNKS, n_heads, Q_CHUNK)
|
||||
torch.testing.assert_close(got[:, :, :n_heads], want, rtol=0, atol=0)
|
||||
assert (got[:, :, n_heads:] == 0).all()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("n_heads", [8, 16, 32])
|
||||
def test_q_interleaved_survives_graph_replay(n_heads: int):
|
||||
"""A captured fused insert refills its padded Q buffer on every replay.
|
||||
|
||||
The op allocates ``q_out`` itself, so the tensor a capture hands back has
|
||||
to keep receiving the new padding when the graph is replayed over changed
|
||||
input -- that is what lets the layer prepare Q inside the captured region
|
||||
rather than in ``forward_mqa``.
|
||||
"""
|
||||
torch.manual_seed(6)
|
||||
device = "cuda"
|
||||
dtype = torch.bfloat16
|
||||
block_size, num_tokens, q_head_padded = 64, 7, 64
|
||||
|
||||
q = torch.randn(num_tokens, n_heads, HEAD_DIM, dtype=dtype, device=device)
|
||||
kv = torch.randn(num_tokens, HEAD_DIM, dtype=dtype, device=device)
|
||||
positions = torch.arange(num_tokens, dtype=torch.int64, device=device)
|
||||
cos_sin_cache = make_cos_sin_cache(4096, ROPE_DIM, torch.float32, device)
|
||||
num_blocks = (num_tokens + block_size - 1) // block_size + 1
|
||||
slot_mapping = torch.arange(num_tokens, dtype=torch.int64, device=device)
|
||||
k_cache = torch.zeros(
|
||||
num_blocks, block_size * V41_HEAD_BYTES, dtype=torch.uint8, device=device
|
||||
)
|
||||
|
||||
def prepare():
|
||||
return torch.ops._C.fused_deepseek_v4_qnorm_rope_kv_rope_quant_insert(
|
||||
q,
|
||||
kv,
|
||||
k_cache,
|
||||
slot_mapping,
|
||||
positions,
|
||||
cos_sin_cache,
|
||||
q_head_padded,
|
||||
1e-6,
|
||||
block_size,
|
||||
False, # apply_q_norm
|
||||
True, # kv_mxfp8
|
||||
False, # apply_q_rope
|
||||
True, # is_q_interleaved
|
||||
)
|
||||
|
||||
# Capture takes its warmup on a side stream.
|
||||
side = torch.cuda.Stream()
|
||||
side.wait_stream(torch.cuda.current_stream())
|
||||
with torch.cuda.stream(side):
|
||||
for _ in range(3):
|
||||
prepare()
|
||||
torch.cuda.current_stream().wait_stream(side)
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph):
|
||||
q_out = prepare()
|
||||
|
||||
for factor in (0.25, -0.75):
|
||||
q.copy_(torch.randn_like(q) * factor)
|
||||
graph.replay()
|
||||
torch.accelerator.synchronize()
|
||||
got = q_out.view(num_tokens, NUM_Q_CHUNKS, q_head_padded, Q_CHUNK)
|
||||
want = q.view(num_tokens, NUM_Q_CHUNKS, n_heads, Q_CHUNK)
|
||||
torch.testing.assert_close(got[:, :, :n_heads], want, rtol=0, atol=0)
|
||||
assert (got[:, :, n_heads:] == 0).all()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("n_heads", [16, 64])
|
||||
@pytest.mark.parametrize("apply_q_norm", [False, True])
|
||||
@pytest.mark.parametrize("apply_q_rope", [False, True])
|
||||
def test_q_interleaved_is_orthogonal_to_norm_and_rope(
|
||||
n_heads: int, apply_q_norm: bool, apply_q_rope: bool
|
||||
):
|
||||
"""The Q layout composes with whatever norm and RoPE it is given.
|
||||
|
||||
A lane owns the same 512-dim slice of its head in either layout -- only
|
||||
the address differs -- so the head-major result permuted into the
|
||||
interleaved layout must equal running the interleaved path on a permuted
|
||||
input, bit for bit, for every norm/RoPE combination.
|
||||
"""
|
||||
from vllm.models.deepseek_v41.common.ops.fused_layout import permute_q_to_fused
|
||||
|
||||
torch.manual_seed(7)
|
||||
device = "cuda"
|
||||
dtype = torch.bfloat16
|
||||
block_size, num_tokens, q_head_padded = 64, 5, 64
|
||||
|
||||
q = torch.randn(num_tokens, n_heads, HEAD_DIM, dtype=dtype, device=device)
|
||||
kv = torch.randn(num_tokens, HEAD_DIM, dtype=dtype, device=device)
|
||||
positions = torch.arange(num_tokens, dtype=torch.int64, device=device)
|
||||
cos_sin_cache = make_cos_sin_cache(4096, ROPE_DIM, torch.float32, device)
|
||||
num_blocks = (num_tokens + block_size - 1) // block_size + 1
|
||||
slot_mapping = torch.arange(num_tokens, dtype=torch.int64, device=device)
|
||||
|
||||
def run(q_in, is_q_interleaved):
|
||||
cache = torch.zeros(
|
||||
num_blocks, block_size * V41_HEAD_BYTES, dtype=torch.uint8, device=device
|
||||
)
|
||||
out = torch.ops._C.fused_deepseek_v4_qnorm_rope_kv_rope_quant_insert(
|
||||
q_in,
|
||||
kv,
|
||||
cache,
|
||||
slot_mapping,
|
||||
positions,
|
||||
cos_sin_cache,
|
||||
q_head_padded,
|
||||
1e-6,
|
||||
block_size,
|
||||
apply_q_norm,
|
||||
True, # kv_mxfp8
|
||||
apply_q_rope,
|
||||
is_q_interleaved,
|
||||
)
|
||||
return out, cache
|
||||
|
||||
head_major, cache_major = run(q, False)
|
||||
interleaved, cache_inter = run(permute_q_to_fused(q), True)
|
||||
|
||||
torch.testing.assert_close(
|
||||
interleaved, permute_q_to_fused(head_major), rtol=0, atol=0
|
||||
)
|
||||
# The Q layout must not leak into the KV half.
|
||||
torch.testing.assert_close(cache_inter, cache_major, rtol=0, atol=0)
|
||||
|
||||
|
||||
# ── Test 2b: DP padding (slot_mapping shorter than q/kv) ─────────────────────
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user