[Perf][DSv4] Fuse inverse RoPE + FP8 quant into FlashInfer sparse MLA (#58621)
Signed-off-by: Yongye Zhu <zyy1102000@gmail.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
29468dde8b
commit
5ff3bbfb08
@@ -511,6 +511,86 @@ def test_flashinfer_mixed_sparse_indices_separates_window_and_padded_width():
|
||||
assert sparse_lens.cpu().tolist() == [padded_width]
|
||||
|
||||
|
||||
def test_flashinfer_mixed_sparse_indices_noncausal_rows_have_no_active_gaps():
|
||||
"""A DSpark non-causal row keeps -1 out of the kernel's active ranges.
|
||||
|
||||
Each draft token's causal window holds its first visible entries, the rest
|
||||
of the block starts at column `window`, and the compressed entries follow
|
||||
directly; `sparse_topk_lens` ends right after them.
|
||||
"""
|
||||
from vllm.models.deepseek_v4.common.ops.cache_utils import (
|
||||
build_flashinfer_mixed_sparse_indices,
|
||||
)
|
||||
|
||||
device = torch.device("cuda")
|
||||
window, width = 4, 8
|
||||
# One request: 1 context token, a 2-token draft block, slots 0..2 visible.
|
||||
decode_swa = torch.full((2, width), -1, dtype=torch.int32, device=device)
|
||||
decode_swa[:, :3] = torch.arange(3, dtype=torch.int32, device=device)
|
||||
compressed = torch.tensor(
|
||||
[[100, 101, -1, -1]] * 2, dtype=torch.int32, device=device
|
||||
)
|
||||
sparse_indices, sparse_lens = build_flashinfer_mixed_sparse_indices(
|
||||
decode_swa_indices=decode_swa,
|
||||
decode_compressed_indices=compressed,
|
||||
decode_compressed_topk_lens=torch.tensor(
|
||||
[2, 2], dtype=torch.int32, device=device
|
||||
),
|
||||
prefill_topk_indices=torch.empty((0, 4), dtype=torch.int32, device=device),
|
||||
query_start_loc=torch.tensor([0, 2], dtype=torch.int32, device=device),
|
||||
seq_lens=torch.tensor([3], dtype=torch.int32, device=device),
|
||||
token_to_req_indices=torch.tensor([0, 0], dtype=torch.int32, device=device),
|
||||
swa_block_table=torch.tensor([[0]], dtype=torch.int32, device=device),
|
||||
swa_block_size=64,
|
||||
compressed_block_table=torch.tensor([[0]], dtype=torch.int32, device=device),
|
||||
compressed_block_size=64,
|
||||
window_size=window,
|
||||
compress_ratio=128,
|
||||
topk=0,
|
||||
)
|
||||
|
||||
assert sparse_indices.cpu().tolist() == [
|
||||
# pos 1: window [0, 1]; slot 2 spills to column 4; compressed after it.
|
||||
[0, 1, -1, -1, 2, 100, 101, -1, -1, -1, -1, -1],
|
||||
# pos 2: window [0, 1, 2]; nothing spills; compressed at column 4.
|
||||
[0, 1, 2, -1, 100, 101, -1, -1, -1, -1, -1, -1],
|
||||
]
|
||||
assert sparse_lens.cpu().tolist() == [7, 6]
|
||||
|
||||
|
||||
def test_flashinfer_mixed_sparse_indices_noncausal_pad_rows_keep_min_length():
|
||||
"""A CUDA-graph pad token of the DSpark graph (a zero-length request) gets
|
||||
a causal pad row's shape instead of negative spill arithmetic."""
|
||||
from vllm.models.deepseek_v4.common.ops.cache_utils import (
|
||||
build_flashinfer_mixed_sparse_indices,
|
||||
)
|
||||
|
||||
device = torch.device("cuda")
|
||||
window, width = 128, 192
|
||||
decode_swa = torch.full((3, width), -1, dtype=torch.int32, device=device)
|
||||
decode_swa[:2, :3] = torch.arange(3, dtype=torch.int32, device=device)
|
||||
sparse_indices, sparse_lens = build_flashinfer_mixed_sparse_indices(
|
||||
decode_swa_indices=decode_swa,
|
||||
decode_compressed_indices=None,
|
||||
decode_compressed_topk_lens=None,
|
||||
prefill_topk_indices=torch.empty((0, 0), dtype=torch.int32, device=device),
|
||||
# Request 1 is the padding: no query tokens, seq_len 0.
|
||||
query_start_loc=torch.tensor([0, 2, 2], dtype=torch.int32, device=device),
|
||||
seq_lens=torch.tensor([3, 0], dtype=torch.int32, device=device),
|
||||
token_to_req_indices=torch.tensor([0, 0, 1], dtype=torch.int32, device=device),
|
||||
swa_block_table=torch.zeros((2, 1), dtype=torch.int32, device=device),
|
||||
swa_block_size=64,
|
||||
compressed_block_table=None,
|
||||
compressed_block_size=64,
|
||||
window_size=window,
|
||||
compress_ratio=1,
|
||||
topk=0,
|
||||
)
|
||||
|
||||
assert sparse_lens.cpu().tolist() == [129, 128, 128]
|
||||
assert (sparse_indices[2] == -1).all()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", ["deepseek_v4", "deepseek_v41"])
|
||||
@pytest.mark.parametrize("page_padding", [0, 64])
|
||||
def test_flashinfer_sparse_forward_reads_packed_kv_rows(model, page_padding):
|
||||
@@ -565,3 +645,198 @@ def test_flashinfer_sparse_forward_reads_packed_kv_rows(model, page_padding):
|
||||
attn._forward(query, None, cache, metadata, None, True, output)
|
||||
expected = torch.tensor([2, 3, 3], dtype=output.dtype, device=device)
|
||||
torch.testing.assert_close(output, expected[:, None, None].expand_as(output))
|
||||
|
||||
|
||||
def _make_rope_quant_attn(mod, num_rows: int, device: str):
|
||||
"""A bare DSv4 FlashInfer layer set up for RopeQuant: 128 heads, fp8 KV."""
|
||||
from types import SimpleNamespace
|
||||
|
||||
heads, groups, rank = 128, 16, 256
|
||||
angles = torch.arange(2048, device=device, dtype=torch.float32)[:, None] * 0.05
|
||||
angles = angles + torch.arange(32, device=device)[None] * 0.1
|
||||
attn = object.__new__(mod.DeepseekV4FlashInferMLAAttention)
|
||||
attn.compress_ratio = 1
|
||||
attn.window_size = 128
|
||||
attn.scale = 512**-0.5
|
||||
attn.kv_cache_torch_dtype = torch.float8_e4m3fn
|
||||
attn._flashinfer_fp8_bmm1_scale = attn.scale
|
||||
attn._flashinfer_fp8_bmm2_scale = 1.0
|
||||
attn.attn_sink = torch.linspace(-1, 1, heads, device=device)
|
||||
attn.padded_heads = heads
|
||||
attn.topk_indices_buffer = torch.empty(
|
||||
(num_rows, 0), dtype=torch.int32, device=device
|
||||
)
|
||||
attn.rotary_emb = SimpleNamespace(
|
||||
cos_sin_cache=torch.cat((angles.cos(), angles.sin()), -1).contiguous()
|
||||
)
|
||||
attn.n_local_heads, attn.n_local_groups, attn.o_lora_rank = heads, groups, rank
|
||||
attn.nope_head_dim, attn.rope_head_dim = 448, 64
|
||||
attn._einsum_recipe, attn._tma_aligned_scales = (1, 1, 128), True
|
||||
attn.wo_a = SimpleNamespace(
|
||||
weight=(torch.randn((groups, rank, 4096), device=device) * 0.05).to(
|
||||
torch.float8_e4m3fn
|
||||
),
|
||||
weight_scale=torch.ones((groups, rank, 32), device=device),
|
||||
)
|
||||
attn.wo_b = lambda z: z
|
||||
return attn
|
||||
|
||||
|
||||
def _rope_quant_vs_unfused(attn, q, cache, metadata, positions, num_tokens):
|
||||
"""Run `_forward` unfused and fused; return (bf16 attention, z_unfused, z_fused)."""
|
||||
from vllm.models.deepseek_v4.nvidia.ops.o_proj import rope_quant_attn_out
|
||||
|
||||
heads, groups = attn.n_local_heads, attn.n_local_groups
|
||||
unfused = torch.empty(q.shape, dtype=torch.bfloat16, device=q.device)
|
||||
attn._forward(q, None, cache, metadata, None, True, unfused)
|
||||
fused = rope_quant_attn_out(q.shape[0], q.device)
|
||||
metadata.flashinfer_sparse_index_cache.clear()
|
||||
attn._forward(q, None, cache, metadata, None, True, fused)
|
||||
z_unfused = attn._o_proj(unfused[:num_tokens, :heads], positions)
|
||||
z_fused = attn._o_proj(fused, positions)[:num_tokens]
|
||||
return unfused[:num_tokens], z_unfused, z_fused.view(num_tokens, groups, -1)
|
||||
|
||||
|
||||
def _assert_projects_alike(z_unfused, z_fused):
|
||||
z_unfused = z_unfused.float().view_as(z_fused)
|
||||
rel = (z_fused.float() - z_unfused).norm() / z_unfused.norm()
|
||||
assert rel < 2e-2, rel
|
||||
|
||||
|
||||
def _skip_unless_rope_quant():
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
if not current_platform.is_device_capability_family(100):
|
||||
pytest.skip("Requires FlashInfer TRTLLM sparse MLA on SM100/SM103")
|
||||
|
||||
|
||||
def test_flashinfer_rope_quant_matches_unfused_o_proj():
|
||||
"""RopeQuant's fused output must project like the bf16 + inv-RoPE/quant path.
|
||||
|
||||
One mixed step (two decodes, a three-token prefill) plus CUDA-graph pad rows,
|
||||
so both calls run over the full buffer with un-rebased cum_seq_lens_q.
|
||||
"""
|
||||
from vllm.models.deepseek_v4.nvidia import flashinfer_sparse as mod
|
||||
from vllm.v1.attention.backends.mla.sparse_swa import DeepseekSparseSWAMetadata
|
||||
|
||||
_skip_unless_rope_quant()
|
||||
torch.manual_seed(0)
|
||||
device = "cuda"
|
||||
block_size = 64
|
||||
q_lens, seq_lens = [1, 1, 3], [5, 9, 70]
|
||||
num_tokens, num_rows = sum(q_lens), sum(q_lens) + 3
|
||||
cache = torch.randn((3, block_size, 512), device=device).to(torch.float8_e4m3fn)
|
||||
attn = _make_rope_quant_attn(mod, num_rows, device)
|
||||
|
||||
decode_indices = torch.full((2, 128), -1, dtype=torch.int32, device=device)
|
||||
for i, seq_len in enumerate(seq_lens[:2]):
|
||||
decode_indices[i, :seq_len] = torch.arange(
|
||||
i * block_size, i * block_size + seq_len, device=device
|
||||
)
|
||||
query_start = torch.tensor([0, 1, 2, 5], dtype=torch.int32)
|
||||
metadata = DeepseekSparseSWAMetadata(
|
||||
block_table=torch.tensor([[0], [1], [2]], dtype=torch.int32, device=device),
|
||||
slot_mapping=torch.zeros(num_tokens, dtype=torch.int64, device=device),
|
||||
block_size=block_size,
|
||||
seq_lens=torch.tensor(seq_lens, dtype=torch.int32, device=device),
|
||||
query_start_loc=query_start.to(device),
|
||||
query_start_loc_cpu=query_start,
|
||||
token_to_req_indices=torch.tensor(
|
||||
[0, 1, 2, 2, 2], dtype=torch.int32, device=device
|
||||
),
|
||||
decode_swa_indices=decode_indices,
|
||||
decode_swa_lens=torch.tensor(seq_lens[:2], dtype=torch.int32, device=device),
|
||||
decode_swa_width=128,
|
||||
is_valid_token=torch.ones(num_tokens, dtype=torch.bool, device=device),
|
||||
num_decodes=2,
|
||||
num_prefills=1,
|
||||
num_decode_tokens=2,
|
||||
num_prefill_tokens=3,
|
||||
max_decode_query_len=1,
|
||||
)
|
||||
positions = torch.tensor([4, 8, 67, 68, 69], device=device)
|
||||
q = torch.randn((num_rows, 128, 512), device=device).to(torch.float8_e4m3fn)
|
||||
_, z_unfused, z_fused = _rope_quant_vs_unfused(
|
||||
attn, q, cache, metadata, positions, num_tokens
|
||||
)
|
||||
_assert_projects_alike(z_unfused, z_fused)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("context_len", [0, 20, 127, 900])
|
||||
def test_flashinfer_dspark_noncausal_block_sees_future_tokens(context_len):
|
||||
"""Every DSpark draft token attends to the whole block, fused or not.
|
||||
|
||||
The kernel counts every entry of its active index ranges as a key, so the
|
||||
builder must place the block around each query's causal window without -1
|
||||
gaps, down to an empty context.
|
||||
"""
|
||||
from vllm.models.deepseek_v4.nvidia import flashinfer_sparse as mod
|
||||
from vllm.v1.attention.backends.mla.compressor_utils import (
|
||||
get_dspark_swa_index_width,
|
||||
)
|
||||
from vllm.v1.attention.backends.mla.sparse_swa import DeepseekSparseSWAMetadata
|
||||
|
||||
_skip_unless_rope_quant()
|
||||
torch.manual_seed(0)
|
||||
device = "cuda"
|
||||
block_size, window, draft = 64, 128, 4
|
||||
width = get_dspark_swa_index_width(window, draft)
|
||||
contexts = [context_len, context_len + 37]
|
||||
num_reqs, num_tokens = len(contexts), len(contexts) * draft
|
||||
num_rows = num_tokens + 2
|
||||
pages = -(-(max(contexts) + draft) // block_size)
|
||||
cache = torch.randn((num_reqs * pages, block_size, 512), device=device)
|
||||
cache = cache.clamp_(-1, 1).to(torch.float8_e4m3fn)
|
||||
attn = _make_rope_quant_attn(mod, num_rows, device)
|
||||
attn.attn_sink = torch.full((128,), -float("inf"), device=device)
|
||||
|
||||
# As the DSpark SWA builder lays them out: the trailing window of context
|
||||
# and the whole block, contiguous from column 0.
|
||||
decode_indices = torch.full((num_tokens, width), -1, dtype=torch.int32)
|
||||
visible, positions = [], []
|
||||
for r, ctx in enumerate(contexts):
|
||||
slots = torch.arange(max(ctx - window, 0), ctx + draft) + r * pages * block_size
|
||||
for k in range(draft):
|
||||
decode_indices[r * draft + k, : slots.numel()] = slots.to(torch.int32)
|
||||
visible.append(slots)
|
||||
positions.append(ctx + k)
|
||||
query_start = torch.arange(0, num_tokens + 1, draft, dtype=torch.int32)
|
||||
seq_lens = torch.tensor([c + draft for c in contexts], dtype=torch.int32)
|
||||
metadata = DeepseekSparseSWAMetadata(
|
||||
block_table=torch.arange(
|
||||
num_reqs * pages, dtype=torch.int32, device=device
|
||||
).view(num_reqs, pages),
|
||||
slot_mapping=torch.zeros(num_tokens, dtype=torch.int64, device=device),
|
||||
block_size=block_size,
|
||||
seq_lens=seq_lens.to(device),
|
||||
query_start_loc=query_start.to(device),
|
||||
query_start_loc_cpu=query_start,
|
||||
token_to_req_indices=torch.arange(
|
||||
num_reqs, dtype=torch.int32, device=device
|
||||
).repeat_interleave(draft),
|
||||
decode_swa_indices=decode_indices.to(device),
|
||||
decode_swa_lens=torch.tensor(
|
||||
[v.numel() for v in visible], dtype=torch.int32, device=device
|
||||
),
|
||||
decode_swa_width=width,
|
||||
is_valid_token=torch.ones(num_tokens, dtype=torch.bool, device=device),
|
||||
num_decodes=num_reqs,
|
||||
num_prefills=0,
|
||||
num_decode_tokens=num_tokens,
|
||||
num_prefill_tokens=0,
|
||||
max_decode_query_len=draft,
|
||||
)
|
||||
q = torch.randn((num_rows, 128, 512), device=device).clamp_(-1, 1)
|
||||
q = q.to(torch.float8_e4m3fn)
|
||||
attention, z_unfused, z_fused = _rope_quant_vs_unfused(
|
||||
attn, q, cache, metadata, torch.tensor(positions, device=device), num_tokens
|
||||
)
|
||||
|
||||
keys_all = cache.flatten(0, 1).float()
|
||||
for token, slots in enumerate(visible):
|
||||
keys = keys_all[slots.to(device)]
|
||||
weights = torch.softmax(q[token].float() @ keys.T * attn.scale, -1)
|
||||
torch.testing.assert_close(
|
||||
attention[token].float(), weights @ keys, atol=0.05, rtol=0.05
|
||||
)
|
||||
_assert_projects_alike(z_unfused, z_fused)
|
||||
|
||||
Reference in New Issue
Block a user