[2/N][Attention] Enable masked MHA for sparse MLA prefills (#48770)
Signed-off-by: Matthew Bonanni <mbonanni@redhat.com> Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
co-authored by
OpenAI Codex
parent
864a87febb
commit
82ae4164ee
@@ -862,6 +862,8 @@ def test_split_prefill_chunks(seq_lens, max_buf, expected):
|
||||
PREFILL_BATCH_SPECS = {
|
||||
"short_dense_mha": BatchSpec(seq_lens=[64, 128], query_lens=[64, 128]),
|
||||
"short_context_dense_mha": BatchSpec(seq_lens=[128, 160], query_lens=[64, 32]),
|
||||
"masked_mha": BatchSpec(seq_lens=[256], query_lens=[256]),
|
||||
"masked_mha_chunked_context": BatchSpec(seq_lens=[448, 384], query_lens=[256, 256]),
|
||||
}
|
||||
|
||||
|
||||
@@ -878,7 +880,7 @@ def test_sparse_backend_prefill_correctness(
|
||||
kv_cache_dtype,
|
||||
workspace_init,
|
||||
):
|
||||
"""Test single-pass FA4 dense forward_mha for sparse MLA prefill."""
|
||||
"""Test single-pass dense and masked MHA for sparse MLA prefill."""
|
||||
backend_cls = FlashMLASparseBackend
|
||||
batch_spec = PREFILL_BATCH_SPECS[batch_name]
|
||||
|
||||
@@ -892,7 +894,8 @@ def test_sparse_backend_prefill_correctness(
|
||||
qk_rope_head_dim = 64
|
||||
v_head_dim = 128
|
||||
head_size = kv_lora_rank + qk_rope_head_dim
|
||||
topk_tokens = 512
|
||||
masked_mha = batch_name.startswith("masked_mha")
|
||||
topk_tokens = 200 if masked_mha else 512
|
||||
|
||||
max_seqlen = max(batch_spec.seq_lens)
|
||||
total_cache_tokens = sum(batch_spec.seq_lens)
|
||||
@@ -918,6 +921,7 @@ def test_sparse_backend_prefill_correctness(
|
||||
model_type="deepseek_v2",
|
||||
)
|
||||
model_config.dtype = dtype
|
||||
model_config.model_arch_config.total_num_attention_heads = num_heads
|
||||
model_config.get_num_attention_heads = MethodType(
|
||||
lambda self, parallel_config: num_heads, model_config
|
||||
)
|
||||
@@ -942,13 +946,14 @@ def test_sparse_backend_prefill_correctness(
|
||||
|
||||
# Compute dense reference outputs.
|
||||
total_query_tokens = sum(query_lens)
|
||||
sparse_indices = torch.zeros(
|
||||
total_query_tokens, topk_tokens, dtype=torch.int32, device=device
|
||||
sparse_indices = torch.full(
|
||||
(total_query_tokens, topk_tokens), -1, dtype=torch.int32, device=device
|
||||
)
|
||||
|
||||
all_q, all_kv_c_new, all_k_pe_new = [], [], []
|
||||
kv_c_contexts, k_pe_contexts = [], []
|
||||
reference_outputs = []
|
||||
global_token_idx = 0
|
||||
|
||||
for i in range(batch_spec.batch_size):
|
||||
s_len = seq_lens[i]
|
||||
@@ -981,8 +986,15 @@ def test_sparse_backend_prefill_correctness(
|
||||
for j in range(q_len):
|
||||
attend_end = ctx_len + j + 1
|
||||
q_tok = q_mha[j : j + 1] # (1, H, D_qk)
|
||||
k_attend = k_all[:attend_end] # (N, H, D_qk)
|
||||
v_attend = v_all[:attend_end] # (N, H, D_v)
|
||||
if masked_mha:
|
||||
actual_topk = min(topk_tokens, attend_end)
|
||||
attend_indices = torch.randperm(attend_end, device=device)[:actual_topk]
|
||||
sparse_indices[global_token_idx, :actual_topk] = attend_indices
|
||||
k_attend = k_all[attend_indices]
|
||||
v_attend = v_all[attend_indices]
|
||||
else:
|
||||
k_attend = k_all[:attend_end] # (N, H, D_qk)
|
||||
v_attend = v_all[:attend_end] # (N, H, D_v)
|
||||
|
||||
q_sdpa = q_tok.unsqueeze(0).transpose(1, 2).float()
|
||||
k_sdpa = k_attend.unsqueeze(0).transpose(1, 2).float()
|
||||
@@ -993,6 +1005,7 @@ def test_sparse_backend_prefill_correctness(
|
||||
)
|
||||
out = out.transpose(1, 2).squeeze(0) # (1, H, D_v)
|
||||
reference_outputs.append(out.to(dtype).flatten(start_dim=-2))
|
||||
global_token_idx += 1
|
||||
|
||||
all_q.append(q_mha)
|
||||
all_kv_c_new.append(kv_c_full[ctx_len:])
|
||||
@@ -1047,6 +1060,13 @@ def test_sparse_backend_prefill_correctness(
|
||||
|
||||
builder_cls = backend_cls.get_builder_cls()
|
||||
builder = builder_cls(kv_cache_spec, ["placeholder"], vllm_config, device)
|
||||
if batch_name == "masked_mha_chunked_context":
|
||||
builder.chunked_prefill_workspace_size = block_size * batch_spec.batch_size
|
||||
builder.chunked_prefill_workspace = torch.empty(
|
||||
(builder.chunked_prefill_workspace_size, head_size),
|
||||
dtype=dtype,
|
||||
device=device,
|
||||
)
|
||||
# Drive the queries through the dense-MHA prefill path directly (the routing
|
||||
# threshold would otherwise classify these short queries as MQA decodes).
|
||||
builder.reorder_batch_threshold = 1
|
||||
|
||||
@@ -0,0 +1,29 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
pytest.importorskip("cutlass")
|
||||
|
||||
from vllm.model_executor.layers.attention.sparse_mla_attention import (
|
||||
_build_topk_mask,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
|
||||
def test_build_topk_mask_single_request_matches_generic_path() -> None:
|
||||
topk = torch.tensor(
|
||||
[[0, 31, 32, 63, -1], [1, 64, 127, -1, -1]],
|
||||
dtype=torch.int32,
|
||||
device="cuda",
|
||||
)
|
||||
|
||||
num_words = (128 + 31) // 32
|
||||
single_out = torch.zeros(1, 2, num_words, dtype=torch.int32, device="cuda")
|
||||
generic_out = torch.zeros(2, 1, num_words, dtype=torch.int32, device="cuda")
|
||||
single_req = _build_topk_mask([topk], [2], 2, 128, single_out)
|
||||
generic = _build_topk_mask([topk[:1], topk[1:]], [1, 1], 1, 128, generic_out)
|
||||
|
||||
torch.testing.assert_close(single_req[0, 0], generic[0, 0])
|
||||
torch.testing.assert_close(single_req[0, 1], generic[1, 0])
|
||||
Reference in New Issue
Block a user