[DSA] Wire DeepGEMM sparse MQA logits into the DeepSeek V4.1 indexer (#56254)
Signed-off-by: Jared Wen <w13431838023@gmail.com> Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Fable 5.1
parent
f2aad6aa70
commit
1257512305
@@ -560,3 +560,55 @@ def test_indexer_k_store_roundtrips_through_rocm_gather(block_size, compress_rat
|
||||
torch.testing.assert_close(
|
||||
k_scale[emitted], expected_scale[emitted], rtol=0, atol=0
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("use_cutedsl", [False, True])
|
||||
@pytest.mark.skipif(
|
||||
not (
|
||||
current_platform.is_cuda() and current_platform.is_device_capability_family(100)
|
||||
),
|
||||
reason="MXFP4 indexer cache requires an SM100-family GPU",
|
||||
)
|
||||
@torch.inference_mode()
|
||||
def test_fused_indexer_q_rope_quant_writes_bf16_weights(use_cutedsl):
|
||||
"""The MXFP4 path can emit the per-head weights in bf16 (what DeepGEMM's
|
||||
sparse MQA-logits kernels take) instead of fp32; the values are the fp32
|
||||
result rounded once, so the scoring kernel needs no cast."""
|
||||
if use_cutedsl and not has_cutedsl():
|
||||
pytest.skip("cutedsl (cutlass) not installed")
|
||||
|
||||
device = "cuda"
|
||||
torch.manual_seed(0)
|
||||
num_tokens, n_head = 257, 32
|
||||
q = torch.randn(num_tokens, n_head, HEAD_DIM, dtype=torch.bfloat16, device=device)
|
||||
positions = torch.randint(
|
||||
0, MAX_POS, (num_tokens,), dtype=torch.int64, device=device
|
||||
)
|
||||
cos_sin_cache = torch.randn(MAX_POS, ROPE_DIM, dtype=torch.float32, device=device)
|
||||
weights = torch.randn(num_tokens, n_head, dtype=torch.bfloat16, device=device)
|
||||
softmax_scale, head_scale = HEAD_DIM**-0.5, 0.125 # pow2 head_scale, see above
|
||||
|
||||
_, weights_ref = _reference(
|
||||
positions, q, cos_sin_cache, weights, softmax_scale, head_scale, n_head, True
|
||||
)
|
||||
cutedsl_patch = (
|
||||
mock.patch(
|
||||
"vllm.models.deepseek_v4.common.ops.fused_indexer_q.has_cutedsl",
|
||||
return_value=False,
|
||||
)
|
||||
if not use_cutedsl
|
||||
else contextlib.nullcontext()
|
||||
)
|
||||
with cutedsl_patch:
|
||||
_, weights_fused = fused_indexer_q_rope_quant(
|
||||
positions,
|
||||
q.clone(),
|
||||
cos_sin_cache,
|
||||
weights,
|
||||
softmax_scale,
|
||||
head_scale,
|
||||
use_fp4=True,
|
||||
weights_out_dtype=torch.bfloat16,
|
||||
)
|
||||
assert weights_fused.dtype == torch.bfloat16
|
||||
assert torch.equal(weights_fused, weights_ref.to(torch.bfloat16))
|
||||
|
||||
@@ -450,3 +450,547 @@ def test_candidate_kernels_preserve_packed_bounds_and_padding(
|
||||
keep = valid & (block >= 0) & ((cols - ks[:, None]) // block_size == block)
|
||||
reference = torch.where(keep, 3.0, -torch.inf)
|
||||
torch.testing.assert_close(logits, reference, rtol=0, atol=0)
|
||||
|
||||
|
||||
# --- DeepGEMM sparse-MQA indexer path (V4.1 two-level candidate filtering) ---
|
||||
|
||||
|
||||
def _make_candidates(lens: torch.Tensor, num_candidates: int, cbk: int):
|
||||
"""Production-faithful candidates: top min(avail, K) blocks per row —
|
||||
full coverage for lens <= K*cbk, a fixed-size subset beyond that."""
|
||||
avail = ((lens.cpu() + cbk - 1) // cbk).tolist()
|
||||
rows = []
|
||||
for a in avail:
|
||||
n = min(a, num_candidates)
|
||||
picks = torch.randperm(a)[:n].sort().values[:num_candidates].tolist()
|
||||
rows.append(picks + [-1] * (num_candidates - len(picks)))
|
||||
return torch.tensor(rows, dtype=torch.int32)
|
||||
|
||||
|
||||
def _candidate_token_mask(
|
||||
candidates: torch.Tensor, cbk: int, total: int, ks: torch.Tensor | None = None
|
||||
):
|
||||
"""[rows, total] bool mask of the candidate-covered token positions.
|
||||
|
||||
``ks`` converts request-local candidate blocks to absolute positions
|
||||
(prefill's packed workspace); pass None when already request-local.
|
||||
"""
|
||||
rows = candidates.shape[0]
|
||||
device = candidates.device
|
||||
# Sentinel block keeps -1 padding from scattering onto real columns.
|
||||
width = max(total, (candidates.shape[1] + 1) * cbk)
|
||||
if ks is not None:
|
||||
width += int(ks.max()) + cbk
|
||||
in_cand = torch.zeros(rows, width, dtype=torch.bool, device=device)
|
||||
block_id = torch.where(candidates >= 0, candidates.long(), candidates.shape[1])
|
||||
tok = block_id.unsqueeze(2) * cbk + torch.arange(cbk, device=device)
|
||||
if ks is not None:
|
||||
tok = tok + ks.unsqueeze(1).unsqueeze(2)
|
||||
keep = (candidates >= 0).unsqueeze(2).expand(-1, -1, cbk)
|
||||
in_cand.scatter_(1, tok.flatten(1), keep.flatten(1))
|
||||
return in_cand[:, :total]
|
||||
|
||||
|
||||
def _reference_topk(masked: torch.Tensor, topk: int) -> torch.Tensor:
|
||||
"""Plain ``torch.topk`` over -inf-masked logits; slots past a row's valid
|
||||
count are -1 (the indexer's "no token" sentinel)."""
|
||||
values, indices = masked.topk(topk, dim=1)
|
||||
return torch.where(values > float("-inf"), indices.int(), -1)
|
||||
|
||||
|
||||
def _assert_same_selection(
|
||||
logits: torch.Tensor,
|
||||
got: torch.Tensor,
|
||||
expected: torch.Tensor,
|
||||
row_starts: torch.Tensor | None = None,
|
||||
) -> None:
|
||||
"""Both top-k index sets must pick the same multiset of logit values per
|
||||
row (-1 padding must match too). Indices themselves may differ: the
|
||||
top-k kernels break ties among equal values in nondeterministic order."""
|
||||
for r in range(logits.shape[0]):
|
||||
base = int(row_starts[r]) if row_starts is not None else 0
|
||||
g, e = got[r][got[r] >= 0], expected[r][expected[r] >= 0]
|
||||
assert g.numel() == e.numel(), f"row {r}: {g.numel()} vs {e.numel()} selected"
|
||||
vg = logits[r][(base + g).long()].sort().values
|
||||
ve = logits[r][(base + e).long()].sort().values
|
||||
assert torch.equal(vg, ve), f"row {r}: selected values differ"
|
||||
|
||||
|
||||
def _sparse_scratch(rows: int, num_sparse: int, topk: int) -> dict:
|
||||
"""Caller-owned buffers the sparse path writes into (the metadata builder
|
||||
owns these in production)."""
|
||||
i32 = dict(dtype=torch.int32, device="cuda")
|
||||
return dict(
|
||||
sparse_indices=torch.zeros(rows, num_sparse, **i32),
|
||||
end=torch.zeros(rows, **i32),
|
||||
col_indices=torch.zeros(rows, topk, **i32),
|
||||
)
|
||||
|
||||
|
||||
def _expand_candidates_reference(
|
||||
candidates: torch.Tensor, ks: torch.Tensor, ke: torch.Tensor, cbk: int, sbk: int
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""PyTorch reference for `candidate_blocks_to_sparse_indices`: expand
|
||||
each candidate block into ``cbk // sbk`` sparse blocks, keep those inside
|
||||
``[0, ke - ks)``, sort ascending, anchor at ``ks // sbk``, pad by
|
||||
repeating the last valid block, and count the valid sparse columns."""
|
||||
ratio = cbk // sbk
|
||||
rows, num_candidates = candidates.shape
|
||||
num_sparse = num_candidates * ratio
|
||||
big = torch.iinfo(torch.int32).max
|
||||
offsets = torch.arange(ratio, device=candidates.device, dtype=torch.int32)
|
||||
blocks = (candidates.unsqueeze(-1) * ratio + offsets).flatten(1)
|
||||
max_block = ((ke - ks).clamp(min=0) + sbk - 1) // sbk
|
||||
valid = (blocks >= 0) & (blocks < max_block.unsqueeze(1))
|
||||
sorted_blocks, _ = torch.where(valid, blocks, big).sort(dim=1)
|
||||
n_valid = (sorted_blocks != big).sum(dim=1, dtype=torch.int32)
|
||||
base = (ks // sbk).unsqueeze(1)
|
||||
kernel_blocks = torch.where(sorted_blocks != big, sorted_blocks, 0) + base
|
||||
last_slot = (n_valid - 1).clamp(min=0).unsqueeze(1).long()
|
||||
last_block = kernel_blocks.gather(1, last_slot)
|
||||
pad = torch.arange(num_sparse, device=candidates.device) >= n_valid.unsqueeze(1)
|
||||
sparse_indices = torch.where(pad, last_block, kernel_blocks)
|
||||
last_start = last_block.squeeze(1) * sbk + ks % sbk
|
||||
last_fill = (ke - last_start).clamp(min=1, max=sbk)
|
||||
end = torch.where(n_valid > 0, (n_valid - 1) * sbk + last_fill, 0)
|
||||
return sparse_indices.int(), end.int()
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="Triton kernel")
|
||||
def test_candidate_blocks_to_sparse_indices_math():
|
||||
"""Candidate->sparse-block expansion: ks anchoring, range filtering,
|
||||
repeat-last padding, and the per-row valid column count (``end``)."""
|
||||
from vllm.model_executor.kernels.attention.dsa.sparse_mqa_logits import (
|
||||
candidate_blocks_to_sparse_indices,
|
||||
)
|
||||
|
||||
# Rows cover: exactly one block, an unaligned ks with a partial tail
|
||||
# block, an empty row, duplicates / -1 padding / out-of-range garbage
|
||||
# (warmup runs touch uninitialized buffers), and int32-overflow junk.
|
||||
ks = torch.tensor([64, 13, 100, 8, 0], dtype=torch.int32, device="cuda")
|
||||
ke = torch.tensor([72, 30, 100, 40, 1000], dtype=torch.int32, device="cuda")
|
||||
candidates = torch.tensor(
|
||||
[
|
||||
[0, -1, -1, -1],
|
||||
[2, 0, -1, -1],
|
||||
[3, -1, -1, -1],
|
||||
[1, 1, 7, 0],
|
||||
[2**30, -(2**30), 0, -1],
|
||||
],
|
||||
dtype=torch.int32,
|
||||
device="cuda",
|
||||
)
|
||||
indices, end = candidate_blocks_to_sparse_indices(
|
||||
candidates, ks, ke, candidate_block_size=8, sparse_block_kv=8
|
||||
)
|
||||
assert indices.tolist() == [
|
||||
[8, 8, 8, 8],
|
||||
[1, 3, 3, 3],
|
||||
[12, 12, 12, 12],
|
||||
[1, 2, 2, 2],
|
||||
[0, 0, 0, 0],
|
||||
]
|
||||
# end: row 1's last block covers [29, 37) but ke = 30 -> one valid token;
|
||||
# row 3 keeps the duplicated block (production candidates are unique).
|
||||
assert end.tolist() == [8, 9, 0, 24, 8]
|
||||
|
||||
# Wider candidate blocks expand into multiple sparse blocks.
|
||||
idx2, end2 = candidate_blocks_to_sparse_indices(
|
||||
torch.tensor([[1, 0]], dtype=torch.int32, device="cuda"),
|
||||
torch.tensor([0], dtype=torch.int32, device="cuda"),
|
||||
torch.tensor([48], dtype=torch.int32, device="cuda"),
|
||||
candidate_block_size=16,
|
||||
sparse_block_kv=8,
|
||||
)
|
||||
assert idx2.tolist() == [[0, 1, 2, 3]]
|
||||
assert end2.tolist() == [32]
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="Triton kernel")
|
||||
@pytest.mark.parametrize("cbk,sbk", [(8, 8), (16, 8), (16, 16)])
|
||||
def test_candidate_blocks_to_sparse_indices_matches_reference(cbk: int, sbk: int):
|
||||
"""The Triton expansion equals the PyTorch reference on production-shaped
|
||||
random inputs, including strided (column-sliced) candidate rows and
|
||||
strided output buffers."""
|
||||
from vllm.model_executor.kernels.attention.dsa.sparse_mqa_logits import (
|
||||
candidate_blocks_to_sparse_indices,
|
||||
)
|
||||
|
||||
torch.manual_seed(0)
|
||||
rows, num_candidates = 41, 2048
|
||||
ratio = cbk // sbk
|
||||
lens = torch.randint(1, 40000, (rows,), dtype=torch.int32)
|
||||
lens[0], lens[1] = 1, 3
|
||||
ks = torch.zeros(rows, dtype=torch.int32)
|
||||
ks[1:] = lens.cumsum(0)[:-1]
|
||||
ks, ke = ks.cuda(), (ks + lens).cuda()
|
||||
candidates = _make_candidates(lens, num_candidates, cbk).cuda()
|
||||
# Row 2 gets out-of-range garbage past the padding, as a CUDA-graph
|
||||
# warm-up on uninitialized buffers would.
|
||||
candidates[2, -3:] = torch.tensor([2**30, -(2**29), 10**6], dtype=torch.int32)
|
||||
|
||||
ref_indices, ref_end = _expand_candidates_reference(candidates, ks, ke, cbk, sbk)
|
||||
|
||||
cand_buf = torch.full((rows, num_candidates + 64), -7, dtype=torch.int32).cuda()
|
||||
cand_buf[:, :num_candidates] = candidates
|
||||
si_buf = torch.zeros(rows, num_candidates * ratio + 32, dtype=torch.int32).cuda()
|
||||
end_buf = torch.zeros(rows * 2, dtype=torch.int32, device="cuda")
|
||||
ks_buf = torch.zeros(rows * 2, dtype=torch.int32, device="cuda")
|
||||
ks_buf[::2] = ks
|
||||
got_indices, got_end = candidate_blocks_to_sparse_indices(
|
||||
cand_buf[:, :num_candidates],
|
||||
ks_buf[::2],
|
||||
ke,
|
||||
cbk,
|
||||
sbk,
|
||||
out=(si_buf[:, : num_candidates * ratio], end_buf[::2]),
|
||||
)
|
||||
assert torch.equal(got_indices, ref_indices)
|
||||
assert torch.equal(got_end, ref_end)
|
||||
assert torch.equal(
|
||||
cand_buf[:, num_candidates:], torch.full_like(cand_buf[:, num_candidates:], -7)
|
||||
)
|
||||
assert not si_buf[:, num_candidates * ratio :].any()
|
||||
assert not end_buf[1::2].any()
|
||||
|
||||
|
||||
def _skip_unless_sm100_sparse_kernels():
|
||||
if not torch.cuda.is_available() or torch.cuda.get_device_capability()[0] != 10:
|
||||
pytest.skip("SM100 required")
|
||||
deep_gemm = pytest.importorskip("deep_gemm")
|
||||
if not hasattr(deep_gemm, "fp8_fp4_sparse_mqa_logits"):
|
||||
pytest.skip("DeepGEMM >= 2.8 required for sparse MQA logits")
|
||||
from vllm.model_executor.kernels.attention.dsa.sparse_mqa_logits import (
|
||||
has_deep_select,
|
||||
)
|
||||
|
||||
if not has_deep_select():
|
||||
pytest.skip("DeepSelect extension (vllm._deepselect_C) required")
|
||||
return deep_gemm
|
||||
|
||||
|
||||
def _quant_fp4(x: torch.Tensor):
|
||||
from deep_gemm.utils import per_token_cast_to_fp4
|
||||
|
||||
return per_token_cast_to_fp4(x, use_ue8m0=True, gran_k=32, use_packed_ue8m0=True)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("capture", [False, True])
|
||||
def test_sparse_mqa_logits_prefill_matches_dense_masked(capture: bool):
|
||||
"""The sparse-MQA candidate path selects exactly the same logit values as
|
||||
``torch.topk`` over dense bf16 logits masked to the candidates (the two
|
||||
DeepGEMM kernels agree bitwise on candidate columns), incl. under CUDA
|
||||
graph capture, and a later layer reusing the step's expansion/schedule
|
||||
selects the same values."""
|
||||
deep_gemm = _skip_unless_sm100_sparse_kernels()
|
||||
from vllm.model_executor.kernels.attention.dsa.sparse_mqa_logits import (
|
||||
sparse_mqa_logits_prefill_chunk,
|
||||
)
|
||||
|
||||
torch.manual_seed(0)
|
||||
rows, num_heads, head_dim = 37, 32, 128
|
||||
cbk, topk, num_candidates = 8, 512, 2048 # V4.1-Flash production values
|
||||
lens = torch.randint(1, 30000, (rows,), dtype=torch.int32)
|
||||
lens[0], lens[1], lens[2] = 1, 5, 2048 # single token, sub-block, long
|
||||
ks = torch.zeros(rows, dtype=torch.int32)
|
||||
ks[1:] = lens.cumsum(0)[:-1]
|
||||
ks, ke, lens = ks.cuda(), (ks + lens).cuda(), lens.cuda()
|
||||
total = int(lens.sum())
|
||||
q_fp, q_sf = _quant_fp4(
|
||||
torch.randn(rows * num_heads, head_dim, device="cuda", dtype=torch.bfloat16)
|
||||
)
|
||||
k_fp, k_sf = _quant_fp4(
|
||||
torch.randn(total, head_dim, device="cuda", dtype=torch.bfloat16)
|
||||
)
|
||||
weights = torch.randn(rows, num_heads, device="cuda", dtype=torch.bfloat16)
|
||||
candidates = _make_candidates(lens, num_candidates, cbk).cuda()
|
||||
|
||||
# Reference: dense bf16 logits masked to the candidates, then torch.topk.
|
||||
dense = deep_gemm.fp8_fp4_mqa_logits(
|
||||
(
|
||||
q_fp.view(torch.int8).view(rows, num_heads, head_dim // 2),
|
||||
q_sf.view(rows, num_heads),
|
||||
),
|
||||
(k_fp.view(torch.int8), k_sf.view(total)),
|
||||
weights,
|
||||
ks,
|
||||
ke,
|
||||
clean_logits=False,
|
||||
logits_dtype=torch.bfloat16,
|
||||
)
|
||||
cols = torch.arange(total, device="cuda")
|
||||
in_range = (cols - ks.unsqueeze(1) >= 0) & (
|
||||
cols - ks.unsqueeze(1) < lens.unsqueeze(1)
|
||||
)
|
||||
masked = dense.masked_fill(
|
||||
~(in_range & _candidate_token_mask(candidates, cbk, total, ks)),
|
||||
float("-inf"),
|
||||
).float()
|
||||
ref_abs = _reference_topk(masked, topk) # packed-workspace positions
|
||||
ref = torch.where(ref_abs >= 0, ref_abs - ks.unsqueeze(1), -1)
|
||||
|
||||
out = torch.full((rows, topk), -1, dtype=torch.int32, device="cuda")
|
||||
scratch = _sparse_scratch(rows, num_candidates, topk)
|
||||
|
||||
def run_sparse(kernel_metadata=None):
|
||||
return sparse_mqa_logits_prefill_chunk(
|
||||
q_fp.view(torch.int8).view(rows, num_heads, head_dim // 2),
|
||||
q_sf.view(rows, num_heads),
|
||||
k_fp.view(torch.int8),
|
||||
k_sf.view(total),
|
||||
weights,
|
||||
ks,
|
||||
ke,
|
||||
candidates,
|
||||
cbk,
|
||||
cbk,
|
||||
topk,
|
||||
out,
|
||||
kernel_metadata=kernel_metadata,
|
||||
**scratch,
|
||||
)
|
||||
|
||||
if capture:
|
||||
stream = torch.cuda.Stream()
|
||||
stream.wait_stream(torch.cuda.current_stream())
|
||||
with torch.cuda.stream(stream):
|
||||
run_sparse() # warm up JIT + the DeepGEMM workspace pre-capture
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph, stream=stream):
|
||||
run_sparse()
|
||||
out.fill_(-1)
|
||||
graph.replay()
|
||||
else:
|
||||
kernel_metadata = run_sparse()
|
||||
_assert_same_selection(dense, out, ref, row_starts=ks)
|
||||
out.fill_(-1)
|
||||
run_sparse(kernel_metadata)
|
||||
|
||||
_assert_same_selection(dense, out, ref, row_starts=ks)
|
||||
|
||||
|
||||
def _build_fp4_paged_cache(req_lens: torch.Tensor, page_kv: int, head_dim: int):
|
||||
"""MXFP4 paged cache in DeepGEMM's fused layout: per page, all tokens'
|
||||
fp4 data first, then their packed UE8M0 scales."""
|
||||
device = req_lens.device
|
||||
pages_per = (req_lens + page_kv - 1) // page_kv
|
||||
num_pages, max_pages = int(pages_per.sum()), int(pages_per.max())
|
||||
kv_cache = torch.zeros(
|
||||
num_pages, page_kv, 1, head_dim // 2 + 4, dtype=torch.uint8, device=device
|
||||
)
|
||||
block_table = torch.zeros(
|
||||
len(req_lens), max_pages, dtype=torch.int32, device=device
|
||||
)
|
||||
page_pool = torch.randperm(num_pages, device=device)
|
||||
offset = 0
|
||||
for b in range(len(req_lens)):
|
||||
n = int(pages_per[b])
|
||||
block_table[b, :n] = page_pool[offset : offset + n].to(torch.int32)
|
||||
offset += n
|
||||
k_fp, k_sf = _quant_fp4(
|
||||
torch.randn(int(req_lens[b]), head_dim, device=device, dtype=torch.bfloat16)
|
||||
)
|
||||
rows68 = torch.zeros(n * page_kv, 68, dtype=torch.uint8, device=device)
|
||||
rows68[: int(req_lens[b]), : head_dim // 2] = k_fp.view(torch.uint8)
|
||||
rows68[: int(req_lens[b]), head_dim // 2 :] = k_sf.view(torch.uint8).view(-1, 4)
|
||||
by_page = rows68.view(n, page_kv, 68)
|
||||
split = torch.cat(
|
||||
[by_page[..., :64].reshape(n, -1), by_page[..., 64:].reshape(n, -1)], dim=1
|
||||
).view(n, page_kv, 1, head_dim // 2 + 4)
|
||||
kv_cache[block_table[b, :n].long()] = split
|
||||
return kv_cache, block_table
|
||||
|
||||
|
||||
@pytest.mark.parametrize("next_n", [1, 2])
|
||||
@pytest.mark.parametrize("capture", [False, True])
|
||||
def test_sparse_mqa_logits_paged_matches_dense_masked(next_n: int, capture: bool):
|
||||
"""Paged sparse-MQA decode selects exactly the values ``torch.topk`` picks
|
||||
from the dense bf16 masked logits, incl. MTP rows (next_n > 1 flattened to
|
||||
one query row each), CUDA graph capture/replay and schedule reuse by a
|
||||
later layer."""
|
||||
deep_gemm = _skip_unless_sm100_sparse_kernels()
|
||||
from vllm.model_executor.kernels.attention.dsa.sparse_mqa_logits import (
|
||||
sparse_mqa_logits_paged_decode,
|
||||
)
|
||||
|
||||
torch.manual_seed(0)
|
||||
batch, num_heads, head_dim, page_kv = 9, 32, 128, 128
|
||||
cbk, topk, num_candidates = 8, 512, 2048
|
||||
rows = batch * next_n
|
||||
req_lens = torch.randint(1, 30000, (batch,), dtype=torch.int32, device="cuda")
|
||||
req_lens[0], req_lens[1] = 1, 30000 # tiny row + a filtered (subset) row
|
||||
# Row (b, j) sees L_b - next_n + j + 1 tokens (native MTP layout).
|
||||
row_lens = (
|
||||
(req_lens.unsqueeze(1) - next_n + torch.arange(1, next_n + 1, device="cuda"))
|
||||
.clamp(min=0)
|
||||
.to(torch.int32)
|
||||
)
|
||||
kv_cache, block_table = _build_fp4_paged_cache(req_lens, page_kv, head_dim)
|
||||
assert kv_cache.stride(0) % 512 == 0 # the paged sparse kernel requires it
|
||||
|
||||
q_fp, q_sf = _quant_fp4(
|
||||
torch.randn(rows * num_heads, head_dim, device="cuda", dtype=torch.bfloat16)
|
||||
)
|
||||
q = q_fp.view(torch.int8).view(batch, next_n, num_heads, head_dim // 2)
|
||||
q_scale = q_sf.view(batch, next_n, num_heads)
|
||||
weights = torch.randn(rows, num_heads, device="cuda", dtype=torch.bfloat16)
|
||||
candidates = _make_candidates(row_lens.flatten(), num_candidates, cbk).cuda()
|
||||
|
||||
def reference_topk():
|
||||
context_lens_2d = row_lens.view(batch, next_n)
|
||||
sched = deep_gemm.get_paged_mqa_logits_metadata(
|
||||
context_lens_2d, page_kv, deep_gemm.get_num_sms(), None
|
||||
)
|
||||
logits = deep_gemm.fp8_fp4_paged_mqa_logits(
|
||||
(q, q_scale),
|
||||
kv_cache,
|
||||
weights,
|
||||
context_lens_2d,
|
||||
block_table,
|
||||
sched,
|
||||
int(req_lens.max()),
|
||||
False,
|
||||
torch.bfloat16,
|
||||
None,
|
||||
)
|
||||
lens_flat = row_lens.flatten()
|
||||
cols = torch.arange(logits.shape[1], device="cuda")
|
||||
masked = logits.masked_fill(
|
||||
~(
|
||||
(cols.unsqueeze(0) < lens_flat.unsqueeze(1))
|
||||
& _candidate_token_mask(candidates, cbk, logits.shape[1])
|
||||
),
|
||||
float("-inf"),
|
||||
).float()
|
||||
return _reference_topk(masked, topk), logits
|
||||
|
||||
expected, dense_logits = reference_topk()
|
||||
out = torch.full((rows, topk), -1, dtype=torch.int32, device="cuda")
|
||||
|
||||
# The metadata builder flattens every query to one row: per-row context
|
||||
# lengths, an expanded block table and a row -> request map.
|
||||
context_lens = row_lens.flatten().contiguous()
|
||||
row_block_table = block_table.repeat_interleave(next_n, dim=0)
|
||||
row_indices = torch.arange(batch, dtype=torch.int32, device="cuda")
|
||||
row_indices = row_indices.repeat_interleave(next_n)
|
||||
row_ks = torch.zeros(rows, dtype=torch.int32, device="cuda")
|
||||
scratch = _sparse_scratch(rows, num_candidates, topk)
|
||||
|
||||
def run_sparse(o: torch.Tensor, kernel_metadata=None):
|
||||
return sparse_mqa_logits_paged_decode(
|
||||
q.view(rows, 1, num_heads, head_dim // 2),
|
||||
q_scale.view(rows, 1, num_heads),
|
||||
kv_cache,
|
||||
weights,
|
||||
context_lens,
|
||||
row_block_table,
|
||||
row_indices,
|
||||
candidates,
|
||||
cbk,
|
||||
cbk,
|
||||
topk,
|
||||
o,
|
||||
row_ks=row_ks,
|
||||
kernel_metadata=kernel_metadata,
|
||||
**scratch,
|
||||
)
|
||||
|
||||
if capture:
|
||||
stream = torch.cuda.Stream()
|
||||
stream.wait_stream(torch.cuda.current_stream())
|
||||
with torch.cuda.stream(stream):
|
||||
run_sparse(out)
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph, stream=stream):
|
||||
run_sparse(out)
|
||||
out.fill_(-1)
|
||||
graph.replay()
|
||||
else:
|
||||
kernel_metadata = run_sparse(out)
|
||||
_assert_same_selection(dense_logits, out, expected)
|
||||
out.fill_(-1)
|
||||
run_sparse(out, kernel_metadata)
|
||||
|
||||
_assert_same_selection(dense_logits, out, expected)
|
||||
|
||||
|
||||
def test_sparse_mqa_logits_accuracy_vs_torch():
|
||||
"""Sparse-kernel numerics vs a PyTorch fp32 reference on the dequantized
|
||||
MXFP4 inputs."""
|
||||
_skip_unless_sm100_sparse_kernels()
|
||||
from deep_gemm.utils import cast_back_from_fp4
|
||||
|
||||
from vllm.model_executor.kernels.attention.dsa.sparse_mqa_logits import (
|
||||
candidate_blocks_to_sparse_indices,
|
||||
)
|
||||
from vllm.utils.deep_gemm import (
|
||||
fp8_fp4_sparse_mqa_logits,
|
||||
get_sparse_mqa_logits_metadata,
|
||||
)
|
||||
|
||||
torch.manual_seed(0)
|
||||
rows, num_heads, head_dim = 17, 32, 128
|
||||
sbk, num_candidates = 8, 2048
|
||||
lens = torch.randint(1, 1500, (rows,), dtype=torch.int32)
|
||||
lens[0] = 1
|
||||
ks = torch.zeros(rows, dtype=torch.int32)
|
||||
ks[1:] = lens.cumsum(0)[:-1]
|
||||
ks, ke, lens = ks.cuda(), (ks + lens).cuda(), lens.cuda()
|
||||
total = int(lens.sum())
|
||||
q_fp, q_sf = _quant_fp4(
|
||||
torch.randn(rows * num_heads, head_dim, device="cuda", dtype=torch.bfloat16)
|
||||
)
|
||||
k_fp, k_sf = _quant_fp4(
|
||||
torch.randn(total, head_dim, device="cuda", dtype=torch.bfloat16)
|
||||
)
|
||||
weights = torch.randn(rows, num_heads, device="cuda", dtype=torch.bfloat16)
|
||||
candidates = _make_candidates(lens, num_candidates, sbk).cuda()
|
||||
|
||||
sparse_idx, end = candidate_blocks_to_sparse_indices(candidates, ks, ke, sbk, sbk)
|
||||
metadata = get_sparse_mqa_logits_metadata(
|
||||
ks, ke, total, sparse_idx, torch.int8, sbk
|
||||
)
|
||||
logits = fp8_fp4_sparse_mqa_logits(
|
||||
(
|
||||
q_fp.view(torch.int8).view(rows, num_heads, head_dim // 2),
|
||||
q_sf.view(rows, num_heads),
|
||||
),
|
||||
(k_fp.view(torch.int8), k_sf.view(total)),
|
||||
weights,
|
||||
metadata,
|
||||
sparse_idx.shape[1],
|
||||
sbk,
|
||||
)
|
||||
|
||||
# PyTorch fp32 reference on the dequantized inputs.
|
||||
q_deq = cast_back_from_fp4(q_fp, q_sf, gran_k=32, use_packed_ue8m0=True)
|
||||
k_deq = cast_back_from_fp4(k_fp, k_sf, gran_k=32, use_packed_ue8m0=True)
|
||||
dots = torch.einsum(
|
||||
"rhd,td->rht",
|
||||
q_deq.view(rows, num_heads, head_dim).float(),
|
||||
k_deq.float(),
|
||||
)
|
||||
ref = (dots.relu() * weights.float().unsqueeze(-1)).sum(1) # [rows, total]
|
||||
|
||||
# Compare on valid sparse columns (absolute packed-workspace positions).
|
||||
width = sparse_idx.shape[1] * sbk
|
||||
cols = torch.arange(width, device="cuda")
|
||||
valid = cols.unsqueeze(0) < end.unsqueeze(1).long()
|
||||
pos = (
|
||||
sparse_idx.repeat_interleave(sbk, dim=1).long() * sbk
|
||||
+ (ks % sbk).unsqueeze(1)
|
||||
+ (cols % sbk).unsqueeze(0)
|
||||
).clamp(min=0, max=total - 1)
|
||||
abs_err = (logits.float() - ref.gather(1, pos)).abs()[valid]
|
||||
row_scale = ref.abs().amax(dim=1)
|
||||
norm_err = ((logits.float() - ref.gather(1, pos)).abs() / row_scale.unsqueeze(1))[
|
||||
valid
|
||||
]
|
||||
print(
|
||||
f"sparse vs torch fp32: max_abs={abs_err.max().item():.4f}; "
|
||||
f"rowmax-normalized p50={norm_err.median().item():.5f} "
|
||||
f"p99={norm_err.quantile(0.99).item():.5f} max={norm_err.max().item():.5f}"
|
||||
)
|
||||
# bf16-level numerics: far below the value gaps that change top-k selection.
|
||||
assert norm_err.quantile(0.99).item() < 1e-2
|
||||
assert norm_err.max().item() < 2e-2
|
||||
|
||||
Reference in New Issue
Block a user