[Perf][DSv4] Fuse inverse RoPE + FP8 quant into FlashInfer sparse MLA (#58621)

Signed-off-by: Yongye Zhu <zyy1102000@gmail.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Yongye Zhu
2026-09-24 23:26:50 -07:00
committed by GitHub
co-authored by Claude Opus 5.5
parent 29468dde8b
commit 5ff3bbfb08
5 changed files with 587 additions and 108 deletions
@@ -511,6 +511,86 @@ def test_flashinfer_mixed_sparse_indices_separates_window_and_padded_width():
assert sparse_lens.cpu().tolist() == [padded_width]
def test_flashinfer_mixed_sparse_indices_noncausal_rows_have_no_active_gaps():
"""A DSpark non-causal row keeps -1 out of the kernel's active ranges.
Each draft token's causal window holds its first visible entries, the rest
of the block starts at column `window`, and the compressed entries follow
directly; `sparse_topk_lens` ends right after them.
"""
from vllm.models.deepseek_v4.common.ops.cache_utils import (
build_flashinfer_mixed_sparse_indices,
)
device = torch.device("cuda")
window, width = 4, 8
# One request: 1 context token, a 2-token draft block, slots 0..2 visible.
decode_swa = torch.full((2, width), -1, dtype=torch.int32, device=device)
decode_swa[:, :3] = torch.arange(3, dtype=torch.int32, device=device)
compressed = torch.tensor(
[[100, 101, -1, -1]] * 2, dtype=torch.int32, device=device
)
sparse_indices, sparse_lens = build_flashinfer_mixed_sparse_indices(
decode_swa_indices=decode_swa,
decode_compressed_indices=compressed,
decode_compressed_topk_lens=torch.tensor(
[2, 2], dtype=torch.int32, device=device
),
prefill_topk_indices=torch.empty((0, 4), dtype=torch.int32, device=device),
query_start_loc=torch.tensor([0, 2], dtype=torch.int32, device=device),
seq_lens=torch.tensor([3], dtype=torch.int32, device=device),
token_to_req_indices=torch.tensor([0, 0], dtype=torch.int32, device=device),
swa_block_table=torch.tensor([[0]], dtype=torch.int32, device=device),
swa_block_size=64,
compressed_block_table=torch.tensor([[0]], dtype=torch.int32, device=device),
compressed_block_size=64,
window_size=window,
compress_ratio=128,
topk=0,
)
assert sparse_indices.cpu().tolist() == [
# pos 1: window [0, 1]; slot 2 spills to column 4; compressed after it.
[0, 1, -1, -1, 2, 100, 101, -1, -1, -1, -1, -1],
# pos 2: window [0, 1, 2]; nothing spills; compressed at column 4.
[0, 1, 2, -1, 100, 101, -1, -1, -1, -1, -1, -1],
]
assert sparse_lens.cpu().tolist() == [7, 6]
def test_flashinfer_mixed_sparse_indices_noncausal_pad_rows_keep_min_length():
"""A CUDA-graph pad token of the DSpark graph (a zero-length request) gets
a causal pad row's shape instead of negative spill arithmetic."""
from vllm.models.deepseek_v4.common.ops.cache_utils import (
build_flashinfer_mixed_sparse_indices,
)
device = torch.device("cuda")
window, width = 128, 192
decode_swa = torch.full((3, width), -1, dtype=torch.int32, device=device)
decode_swa[:2, :3] = torch.arange(3, dtype=torch.int32, device=device)
sparse_indices, sparse_lens = build_flashinfer_mixed_sparse_indices(
decode_swa_indices=decode_swa,
decode_compressed_indices=None,
decode_compressed_topk_lens=None,
prefill_topk_indices=torch.empty((0, 0), dtype=torch.int32, device=device),
# Request 1 is the padding: no query tokens, seq_len 0.
query_start_loc=torch.tensor([0, 2, 2], dtype=torch.int32, device=device),
seq_lens=torch.tensor([3, 0], dtype=torch.int32, device=device),
token_to_req_indices=torch.tensor([0, 0, 1], dtype=torch.int32, device=device),
swa_block_table=torch.zeros((2, 1), dtype=torch.int32, device=device),
swa_block_size=64,
compressed_block_table=None,
compressed_block_size=64,
window_size=window,
compress_ratio=1,
topk=0,
)
assert sparse_lens.cpu().tolist() == [129, 128, 128]
assert (sparse_indices[2] == -1).all()
@pytest.mark.parametrize("model", ["deepseek_v4", "deepseek_v41"])
@pytest.mark.parametrize("page_padding", [0, 64])
def test_flashinfer_sparse_forward_reads_packed_kv_rows(model, page_padding):
@@ -565,3 +645,198 @@ def test_flashinfer_sparse_forward_reads_packed_kv_rows(model, page_padding):
attn._forward(query, None, cache, metadata, None, True, output)
expected = torch.tensor([2, 3, 3], dtype=output.dtype, device=device)
torch.testing.assert_close(output, expected[:, None, None].expand_as(output))
def _make_rope_quant_attn(mod, num_rows: int, device: str):
"""A bare DSv4 FlashInfer layer set up for RopeQuant: 128 heads, fp8 KV."""
from types import SimpleNamespace
heads, groups, rank = 128, 16, 256
angles = torch.arange(2048, device=device, dtype=torch.float32)[:, None] * 0.05
angles = angles + torch.arange(32, device=device)[None] * 0.1
attn = object.__new__(mod.DeepseekV4FlashInferMLAAttention)
attn.compress_ratio = 1
attn.window_size = 128
attn.scale = 512**-0.5
attn.kv_cache_torch_dtype = torch.float8_e4m3fn
attn._flashinfer_fp8_bmm1_scale = attn.scale
attn._flashinfer_fp8_bmm2_scale = 1.0
attn.attn_sink = torch.linspace(-1, 1, heads, device=device)
attn.padded_heads = heads
attn.topk_indices_buffer = torch.empty(
(num_rows, 0), dtype=torch.int32, device=device
)
attn.rotary_emb = SimpleNamespace(
cos_sin_cache=torch.cat((angles.cos(), angles.sin()), -1).contiguous()
)
attn.n_local_heads, attn.n_local_groups, attn.o_lora_rank = heads, groups, rank
attn.nope_head_dim, attn.rope_head_dim = 448, 64
attn._einsum_recipe, attn._tma_aligned_scales = (1, 1, 128), True
attn.wo_a = SimpleNamespace(
weight=(torch.randn((groups, rank, 4096), device=device) * 0.05).to(
torch.float8_e4m3fn
),
weight_scale=torch.ones((groups, rank, 32), device=device),
)
attn.wo_b = lambda z: z
return attn
def _rope_quant_vs_unfused(attn, q, cache, metadata, positions, num_tokens):
"""Run `_forward` unfused and fused; return (bf16 attention, z_unfused, z_fused)."""
from vllm.models.deepseek_v4.nvidia.ops.o_proj import rope_quant_attn_out
heads, groups = attn.n_local_heads, attn.n_local_groups
unfused = torch.empty(q.shape, dtype=torch.bfloat16, device=q.device)
attn._forward(q, None, cache, metadata, None, True, unfused)
fused = rope_quant_attn_out(q.shape[0], q.device)
metadata.flashinfer_sparse_index_cache.clear()
attn._forward(q, None, cache, metadata, None, True, fused)
z_unfused = attn._o_proj(unfused[:num_tokens, :heads], positions)
z_fused = attn._o_proj(fused, positions)[:num_tokens]
return unfused[:num_tokens], z_unfused, z_fused.view(num_tokens, groups, -1)
def _assert_projects_alike(z_unfused, z_fused):
z_unfused = z_unfused.float().view_as(z_fused)
rel = (z_fused.float() - z_unfused).norm() / z_unfused.norm()
assert rel < 2e-2, rel
def _skip_unless_rope_quant():
from vllm.platforms import current_platform
if not current_platform.is_device_capability_family(100):
pytest.skip("Requires FlashInfer TRTLLM sparse MLA on SM100/SM103")
def test_flashinfer_rope_quant_matches_unfused_o_proj():
"""RopeQuant's fused output must project like the bf16 + inv-RoPE/quant path.
One mixed step (two decodes, a three-token prefill) plus CUDA-graph pad rows,
so both calls run over the full buffer with un-rebased cum_seq_lens_q.
"""
from vllm.models.deepseek_v4.nvidia import flashinfer_sparse as mod
from vllm.v1.attention.backends.mla.sparse_swa import DeepseekSparseSWAMetadata
_skip_unless_rope_quant()
torch.manual_seed(0)
device = "cuda"
block_size = 64
q_lens, seq_lens = [1, 1, 3], [5, 9, 70]
num_tokens, num_rows = sum(q_lens), sum(q_lens) + 3
cache = torch.randn((3, block_size, 512), device=device).to(torch.float8_e4m3fn)
attn = _make_rope_quant_attn(mod, num_rows, device)
decode_indices = torch.full((2, 128), -1, dtype=torch.int32, device=device)
for i, seq_len in enumerate(seq_lens[:2]):
decode_indices[i, :seq_len] = torch.arange(
i * block_size, i * block_size + seq_len, device=device
)
query_start = torch.tensor([0, 1, 2, 5], dtype=torch.int32)
metadata = DeepseekSparseSWAMetadata(
block_table=torch.tensor([[0], [1], [2]], dtype=torch.int32, device=device),
slot_mapping=torch.zeros(num_tokens, dtype=torch.int64, device=device),
block_size=block_size,
seq_lens=torch.tensor(seq_lens, dtype=torch.int32, device=device),
query_start_loc=query_start.to(device),
query_start_loc_cpu=query_start,
token_to_req_indices=torch.tensor(
[0, 1, 2, 2, 2], dtype=torch.int32, device=device
),
decode_swa_indices=decode_indices,
decode_swa_lens=torch.tensor(seq_lens[:2], dtype=torch.int32, device=device),
decode_swa_width=128,
is_valid_token=torch.ones(num_tokens, dtype=torch.bool, device=device),
num_decodes=2,
num_prefills=1,
num_decode_tokens=2,
num_prefill_tokens=3,
max_decode_query_len=1,
)
positions = torch.tensor([4, 8, 67, 68, 69], device=device)
q = torch.randn((num_rows, 128, 512), device=device).to(torch.float8_e4m3fn)
_, z_unfused, z_fused = _rope_quant_vs_unfused(
attn, q, cache, metadata, positions, num_tokens
)
_assert_projects_alike(z_unfused, z_fused)
@pytest.mark.parametrize("context_len", [0, 20, 127, 900])
def test_flashinfer_dspark_noncausal_block_sees_future_tokens(context_len):
"""Every DSpark draft token attends to the whole block, fused or not.
The kernel counts every entry of its active index ranges as a key, so the
builder must place the block around each query's causal window without -1
gaps, down to an empty context.
"""
from vllm.models.deepseek_v4.nvidia import flashinfer_sparse as mod
from vllm.v1.attention.backends.mla.compressor_utils import (
get_dspark_swa_index_width,
)
from vllm.v1.attention.backends.mla.sparse_swa import DeepseekSparseSWAMetadata
_skip_unless_rope_quant()
torch.manual_seed(0)
device = "cuda"
block_size, window, draft = 64, 128, 4
width = get_dspark_swa_index_width(window, draft)
contexts = [context_len, context_len + 37]
num_reqs, num_tokens = len(contexts), len(contexts) * draft
num_rows = num_tokens + 2
pages = -(-(max(contexts) + draft) // block_size)
cache = torch.randn((num_reqs * pages, block_size, 512), device=device)
cache = cache.clamp_(-1, 1).to(torch.float8_e4m3fn)
attn = _make_rope_quant_attn(mod, num_rows, device)
attn.attn_sink = torch.full((128,), -float("inf"), device=device)
# As the DSpark SWA builder lays them out: the trailing window of context
# and the whole block, contiguous from column 0.
decode_indices = torch.full((num_tokens, width), -1, dtype=torch.int32)
visible, positions = [], []
for r, ctx in enumerate(contexts):
slots = torch.arange(max(ctx - window, 0), ctx + draft) + r * pages * block_size
for k in range(draft):
decode_indices[r * draft + k, : slots.numel()] = slots.to(torch.int32)
visible.append(slots)
positions.append(ctx + k)
query_start = torch.arange(0, num_tokens + 1, draft, dtype=torch.int32)
seq_lens = torch.tensor([c + draft for c in contexts], dtype=torch.int32)
metadata = DeepseekSparseSWAMetadata(
block_table=torch.arange(
num_reqs * pages, dtype=torch.int32, device=device
).view(num_reqs, pages),
slot_mapping=torch.zeros(num_tokens, dtype=torch.int64, device=device),
block_size=block_size,
seq_lens=seq_lens.to(device),
query_start_loc=query_start.to(device),
query_start_loc_cpu=query_start,
token_to_req_indices=torch.arange(
num_reqs, dtype=torch.int32, device=device
).repeat_interleave(draft),
decode_swa_indices=decode_indices.to(device),
decode_swa_lens=torch.tensor(
[v.numel() for v in visible], dtype=torch.int32, device=device
),
decode_swa_width=width,
is_valid_token=torch.ones(num_tokens, dtype=torch.bool, device=device),
num_decodes=num_reqs,
num_prefills=0,
num_decode_tokens=num_tokens,
num_prefill_tokens=0,
max_decode_query_len=draft,
)
q = torch.randn((num_rows, 128, 512), device=device).clamp_(-1, 1)
q = q.to(torch.float8_e4m3fn)
attention, z_unfused, z_fused = _rope_quant_vs_unfused(
attn, q, cache, metadata, torch.tensor(positions, device=device), num_tokens
)
keys_all = cache.flatten(0, 1).float()
for token, slots in enumerate(visible):
keys = keys_all[slots.to(device)]
weights = torch.softmax(q[token].float() @ keys.T * attn.scale, -1)
torch.testing.assert_close(
attention[token].float(), weights @ keys, atol=0.05, rtol=0.05
)
_assert_projects_alike(z_unfused, z_fused)