[Perf][Attention] Reduce GLM sparse MLA preparation overhead (#57458)
Signed-off-by: Summer Yang <girasoleyang@gmail.com> Co-authored-by: Codex <noreply@openai.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: Jiangyun Zhu <riverclouds.zhu@qq.com>
This commit is contained in:
co-authored by
Codex
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Jiangyun Zhu
parent
c64b15cde5
commit
a369a7becc
@@ -130,69 +130,123 @@ def test_flashinfer_mla_decode(dtype: torch.dtype, bs: int, block_size: int):
|
||||
torch.testing.assert_close(out_ans, out_ref, atol=1e-2, rtol=1e-2)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float8_e4m3fn])
|
||||
@pytest.mark.parametrize("mode", ["tp", "dcp", "hisparse"])
|
||||
@requires_sm10x
|
||||
def test_flashinfer_trtllm_sparse_mla_decode_without_rope():
|
||||
"""The native sparse MLA path supports a zero-width rotary tail."""
|
||||
def test_flashinfer_trtllm_sparse_mla_decode_without_rope(dtype, mode, monkeypatch):
|
||||
"""TP skips preparation; empty DCP contributions and HiSparse outputs vanish."""
|
||||
import vllm.v1.attention.backends.mla.flashinfer_mla_sparse as backend
|
||||
from vllm.v1.attention.backends.mla.index_group import HiSparseMLAIndexGroup
|
||||
|
||||
torch.set_default_device("cuda")
|
||||
torch.manual_seed(42)
|
||||
|
||||
batch_size = 2
|
||||
block_size = 64
|
||||
num_blocks = 4
|
||||
sparse_topk = 128
|
||||
valid_lens = torch.tensor([17, 73], dtype=torch.int32)
|
||||
|
||||
query = torch.randn(
|
||||
batch_size,
|
||||
1,
|
||||
NUM_HEADS,
|
||||
KV_LORA_RANK,
|
||||
dtype=torch.bfloat16,
|
||||
)
|
||||
kv_cache = torch.randn(
|
||||
num_blocks,
|
||||
block_size,
|
||||
KV_LORA_RANK,
|
||||
dtype=torch.bfloat16,
|
||||
)
|
||||
|
||||
batch_size, num_heads, block_size, num_blocks, sparse_topk = 3, 8, 64, 4, 2176
|
||||
valid_lens = [1 if mode == "tp" else 0, 17, 73]
|
||||
counts = torch.tensor(valid_lens, dtype=torch.int32)
|
||||
query = torch.randn(batch_size, 1, num_heads, KV_LORA_RANK).to(dtype)
|
||||
kv_cache = torch.randn(num_blocks, block_size, KV_LORA_RANK).to(dtype)
|
||||
num_slots = num_blocks * block_size
|
||||
slot_tables = torch.stack(
|
||||
[torch.randperm(num_slots)[:sparse_topk] for _ in range(batch_size)]
|
||||
).to(torch.int32)
|
||||
for row, valid_len in zip(slot_tables, valid_lens.tolist()):
|
||||
row[valid_len:] = -1
|
||||
indices = torch.full((batch_size, sparse_topk), -1, dtype=torch.int32)
|
||||
for token, length in enumerate(valid_lens):
|
||||
indices[token, :length] = torch.randperm(num_slots)[:length].int()
|
||||
empty = counts == 0
|
||||
reference_indices = indices.clone()
|
||||
reference_indices[:, 0].masked_fill_(empty, 0)
|
||||
workspace = torch.empty(FLASHINFER_WORKSPACE_BUFFER_SIZE, dtype=torch.int8)
|
||||
|
||||
workspace_buffer = torch.empty(
|
||||
FLASHINFER_WORKSPACE_BUFFER_SIZE,
|
||||
dtype=torch.int8,
|
||||
impl = object.__new__(backend.FlashInferMLASparseImpl)
|
||||
impl.qk_nope_head_dim = QK_NOPE_HEAD_DIM
|
||||
impl.kv_lora_rank = KV_LORA_RANK
|
||||
impl.qk_rope_head_dim = 0
|
||||
impl.index_group = (
|
||||
object.__new__(HiSparseMLAIndexGroup) if mode == "hisparse" else None
|
||||
)
|
||||
out = trtllm_batch_decode_with_kv_cache_mla(
|
||||
impl._workspace_buffer = workspace
|
||||
impl.bmm1_scale = QK_NOPE_HEAD_DIM**-0.5
|
||||
impl.bmm2_scale = 1.0
|
||||
impl.is_nope_mla = True
|
||||
impl.need_to_return_lse_for_decode = mode == "dcp"
|
||||
if mode == "tp":
|
||||
|
||||
def unexpected_helper(*args, **kwargs):
|
||||
pytest.fail("TP must not launch empty-query preparation")
|
||||
|
||||
monkeypatch.setattr(
|
||||
backend, "prepare_sparse_mla_safe_lengths", unexpected_helper
|
||||
)
|
||||
|
||||
out, lse = impl._run_mqa_kernel(query.squeeze(1), kv_cache, indices, counts)
|
||||
reference = trtllm_batch_decode_with_kv_cache_mla(
|
||||
query=query,
|
||||
kv_cache=kv_cache.unsqueeze(1),
|
||||
workspace_buffer=workspace_buffer,
|
||||
workspace_buffer=workspace,
|
||||
qk_nope_head_dim=QK_NOPE_HEAD_DIM,
|
||||
kv_lora_rank=KV_LORA_RANK,
|
||||
qk_rope_head_dim=0,
|
||||
block_tables=slot_tables.unsqueeze(1),
|
||||
seq_lens=valid_lens,
|
||||
block_tables=reference_indices.unsqueeze(1),
|
||||
seq_lens=counts,
|
||||
max_seq_len=sparse_topk,
|
||||
sparse_mla_top_k=sparse_topk,
|
||||
sparse_mla_top_k_lens=valid_lens,
|
||||
sparse_mla_top_k_lens=counts.clamp(min=1),
|
||||
bmm1_scale=QK_NOPE_HEAD_DIM**-0.5,
|
||||
bmm2_scale=1.0,
|
||||
).squeeze(1)
|
||||
return_lse=mode == "dcp",
|
||||
)
|
||||
if mode == "dcp":
|
||||
reference, reference_lse = reference
|
||||
reference_lse = reference_lse.reshape(batch_size, num_heads)
|
||||
reference_lse.masked_fill_(empty[:, None], float("-inf"))
|
||||
torch.testing.assert_close(lse, reference_lse, atol=0, rtol=0)
|
||||
|
||||
flat_cache = kv_cache.view(num_slots, KV_LORA_RANK).float()
|
||||
refs = []
|
||||
for batch_idx, valid_len in enumerate(valid_lens.tolist()):
|
||||
selected_kv = flat_cache[slot_tables[batch_idx, :valid_len].long()]
|
||||
scores = torch.einsum("hd,kd->hk", query[batch_idx, 0].float(), selected_kv)
|
||||
probs = torch.softmax(scores * QK_NOPE_HEAD_DIM**-0.5, dim=-1)
|
||||
refs.append(torch.einsum("hk,kd->hd", probs, selected_kv))
|
||||
ref = torch.stack(refs).to(torch.bfloat16)
|
||||
from vllm.v1.attention.ops.dcp import CPTritonContext, correct_attn_out
|
||||
|
||||
torch.testing.assert_close(out, ref, atol=2e-2, rtol=2e-2)
|
||||
# The peer owns only our empty selections. Check our contribution after
|
||||
# the production merge, without zeroing the local attention output.
|
||||
peer_lse = torch.full_like(lse, float("-inf")).masked_fill(empty[:, None], 0)
|
||||
out, _ = correct_attn_out(
|
||||
out, torch.stack((lse, peer_lse)), cp_rank=0, ctx=CPTritonContext()
|
||||
)
|
||||
else:
|
||||
assert lse is None
|
||||
reference = reference.squeeze(1).masked_fill(empty[:, None, None], 0)
|
||||
torch.testing.assert_close(out, reference, atol=0, rtol=0)
|
||||
|
||||
if dtype == torch.bfloat16:
|
||||
flat_cache = kv_cache.view(num_slots, KV_LORA_RANK).float()
|
||||
refs = []
|
||||
for token, length in enumerate(valid_lens):
|
||||
selected_kv = flat_cache[reference_indices[token, :length].long()]
|
||||
scores = torch.einsum("hd,kd->hk", query[token, 0].float(), selected_kv)
|
||||
probs = torch.softmax(scores * QK_NOPE_HEAD_DIM**-0.5, dim=-1)
|
||||
refs.append(torch.einsum("hk,kd->hd", probs, selected_kv))
|
||||
ref = torch.stack(refs).to(torch.bfloat16)
|
||||
torch.testing.assert_close(out, ref, atol=2e-2, rtol=2e-2)
|
||||
|
||||
|
||||
@requires_sm10x
|
||||
def test_sparse_mla_safe_lengths():
|
||||
from vllm.v1.attention.backends.mla.sparse_utils import (
|
||||
prepare_sparse_mla_safe_lengths,
|
||||
)
|
||||
|
||||
device = "cuda"
|
||||
num_tokens = 259
|
||||
# Include a partial tile and expose accidental writes into adjacent storage.
|
||||
counts_storage = torch.full((num_tokens, 2), 17, dtype=torch.int32, device=device)
|
||||
counts = counts_storage[:, 0]
|
||||
counts[::3] = 0
|
||||
original_counts = counts_storage.clone()
|
||||
empty = counts == 0
|
||||
indices_storage = torch.full(
|
||||
(num_tokens, 256), 42, dtype=torch.int32, device=device
|
||||
)
|
||||
indices = indices_storage[:, ::2]
|
||||
expected_indices = indices_storage.clone()
|
||||
expected_indices[:, 0].masked_fill_(empty, 0)
|
||||
safe = prepare_sparse_mla_safe_lengths(indices, counts)
|
||||
torch.testing.assert_close(safe, counts.clamp(min=1), rtol=0, atol=0)
|
||||
torch.testing.assert_close(indices_storage, expected_indices, rtol=0, atol=0)
|
||||
torch.testing.assert_close(counts_storage, original_counts, rtol=0, atol=0)
|
||||
|
||||
|
||||
@requires_sm90
|
||||
|
||||
@@ -148,3 +148,32 @@ def test_concat_mla_q_values_preserved(num_tokens):
|
||||
assert torch.equal(out_bits[..., :nope_dim], ql_nope_bits)
|
||||
|
||||
assert torch.equal(out_bits[..., nope_dim:], q_pe_bits)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("rope_dim,transposed", [(0, False), (0, True), (64, False)])
|
||||
@pytest.mark.parametrize("enabled", [False, True])
|
||||
def test_concat_mla_q_fp8_nope_and_rope(rope_dim, transposed, enabled):
|
||||
"""FP8 query packing preserves values with empty RoPE and strided BMM output."""
|
||||
from vllm.config import CompilationConfig, VllmConfig, set_current_vllm_config
|
||||
from vllm.model_executor.layers.attention.mla_attention import _DecodeConcatQuantFP8
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import GroupShape
|
||||
|
||||
torch.manual_seed(42)
|
||||
shape = (8, 17, 512) if transposed else (17, 8, 512)
|
||||
ql = torch.randn(shape, dtype=torch.bfloat16, device="cuda") * 128
|
||||
if transposed:
|
||||
ql = ql.transpose(0, 1)
|
||||
qp = torch.randn(17, 8, rope_dim, dtype=torch.bfloat16, device="cuda") * 128
|
||||
scale = torch.tensor(0.3, device="cuda", dtype=torch.float32)
|
||||
joined = torch.cat((ql, qp), dim=-1)
|
||||
expected = (joined.float() * scale.reciprocal()).clamp(-448, 448)
|
||||
expected = expected.to(torch.float8_e4m3fn)
|
||||
custom_ops = ["none"] + (["+mla_decode_concat_quant_fp8"] if enabled else [])
|
||||
config = VllmConfig(
|
||||
compilation_config=CompilationConfig(mode=0, custom_ops=custom_ops)
|
||||
)
|
||||
with set_current_vllm_config(config):
|
||||
quant = _DecodeConcatQuantFP8(static=True, group_shape=GroupShape.PER_TENSOR)
|
||||
actual = quant(ql, qp, scale)
|
||||
torch.testing.assert_close(actual.float(), expected.float(), atol=0, rtol=0)
|
||||
assert actual.is_contiguous()
|
||||
|
||||
@@ -700,9 +700,8 @@ def test_dcp_filter_compacts_valid_slots_for_sparse_kernel(
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="This test requires CUDA")
|
||||
@pytest.mark.parametrize("interleave", [1, 2])
|
||||
@pytest.mark.parametrize("dcp_rank", [0, 1])
|
||||
# 384 is not a power of two, so it exercises the multi-tile atomic allocator
|
||||
# rather than the single-tile path 1024 takes.
|
||||
@pytest.mark.parametrize("num_topk", [1024, 384])
|
||||
# Include padded single-tile compaction and the multi-tile atomic fallback.
|
||||
@pytest.mark.parametrize("num_topk", [1024, 2176, 4224])
|
||||
def test_dcp_filter_compaction_matches_reference(
|
||||
interleave: int, dcp_rank: int, num_topk: int
|
||||
):
|
||||
@@ -729,7 +728,7 @@ def test_dcp_filter_compaction_matches_reference(
|
||||
for r in range(num_rows):
|
||||
# Ids are distinct, so a row holds at most `seq` valid entries.
|
||||
hi = min(num_topk * 3 // 4, seq)
|
||||
n_valid = int(torch.randint(num_topk // 4, hi, (1,)).item())
|
||||
n_valid = int(torch.randint(hi // 3, hi, (1,)).item())
|
||||
perm = torch.randperm(seq, device=device)[:n_valid].to(torch.int32)
|
||||
token_indices[r, :n_valid] = perm
|
||||
|
||||
|
||||
@@ -1663,10 +1663,12 @@ def test_split_indexer_prefill_chunks_single_request_overflow():
|
||||
assert out == expected
|
||||
|
||||
|
||||
# 384 is not a power of two, so it counts via the tiled atomic accumulation
|
||||
# rather than the single-tile path 128 takes.
|
||||
@pytest.mark.parametrize("num_topk_tokens", [128, 384])
|
||||
def test_triton_convert_returns_valid_counts(num_topk_tokens: int):
|
||||
# Power-of-two, GLM's padded tile, and atomic fallback, with reused buffers.
|
||||
@pytest.mark.parametrize(
|
||||
"num_topk_tokens,reuse_buffers",
|
||||
[(128, False), (2176, False), (2176, True), (4224, False), (4224, True)],
|
||||
)
|
||||
def test_triton_convert_returns_valid_counts(num_topk_tokens: int, reuse_buffers: bool):
|
||||
"""Test that return_valid_counts correctly counts non-negative indices."""
|
||||
device = torch.device(DEVICE_TYPE)
|
||||
num_tokens = 8
|
||||
@@ -1679,8 +1681,7 @@ def test_triton_convert_returns_valid_counts(num_topk_tokens: int):
|
||||
num_requests * max_blocks_per_req, dtype=torch.int32, device=device
|
||||
).view(num_requests, max_blocks_per_req)
|
||||
|
||||
# Create token indices with varying numbers of valid entries: half the row,
|
||||
# a quarter of it, the whole row, then a single valid entry -- twice over.
|
||||
# Include partly valid, fully valid, and empty selections for each request.
|
||||
token_indices = torch.full(
|
||||
(num_tokens, num_topk_tokens), -1, dtype=torch.int32, device=device
|
||||
)
|
||||
@@ -1688,7 +1689,7 @@ def test_triton_convert_returns_valid_counts(num_topk_tokens: int):
|
||||
num_topk_tokens // 2,
|
||||
num_topk_tokens // 4,
|
||||
num_topk_tokens,
|
||||
1,
|
||||
0,
|
||||
] * 2
|
||||
expected_valid = []
|
||||
for i in range(num_tokens):
|
||||
@@ -1702,6 +1703,15 @@ def test_triton_convert_returns_valid_counts(num_topk_tokens: int):
|
||||
expected_valid, dtype=torch.int32, device=device
|
||||
)
|
||||
|
||||
buffers = {}
|
||||
if reuse_buffers:
|
||||
buffers = dict(
|
||||
out=torch.full_like(token_indices, -99),
|
||||
valid_counts_out=torch.full(
|
||||
(num_tokens,), 99, dtype=torch.int32, device=device
|
||||
),
|
||||
)
|
||||
|
||||
# Test with return_valid_counts=True
|
||||
result, valid_counts = triton_convert_req_index_to_global_index(
|
||||
req_id,
|
||||
@@ -1710,6 +1720,7 @@ def test_triton_convert_returns_valid_counts(num_topk_tokens: int):
|
||||
BLOCK_SIZE=block_size,
|
||||
NUM_TOPK_TOKENS=num_topk_tokens,
|
||||
return_valid_counts=True,
|
||||
**buffers,
|
||||
)
|
||||
|
||||
torch.testing.assert_close(valid_counts, expected_valid_tensor, rtol=0, atol=0)
|
||||
|
||||
Reference in New Issue
Block a user