[1/N] Add dense MHA path for sparse MLA short sequences (#47327)

Signed-off-by: Matthew Bonanni <mbonanni@redhat.com>
Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Matthew Bonanni
2026-07-14 00:29:56 +00:00
committed by GitHub
co-authored by Claude Opus 4.8
parent 0b0ef8d7eb
commit c4f5cd60da
21 changed files with 1990 additions and 646 deletions
@@ -304,6 +304,23 @@ def _run_sparse_backend_vs_sdpa(
scale=kv_cache_scale,
)
# The sparse builder clones the layer's dense-MHA prefill backend from
# static_forward_context; register a mock layer carrying one.
from vllm.v1.attention.backends.mla.prefill import get_mla_prefill_backend
prefill_backend = get_mla_prefill_backend(vllm_config)(
num_heads=num_heads,
scale=scale,
kv_lora_rank=kv_lora_rank,
qk_nope_head_dim=qk_nope_head_dim,
qk_rope_head_dim=qk_rope_head_dim,
v_head_dim=v_head_dim,
vllm_config=vllm_config,
)
vllm_config.compilation_config.static_forward_context["placeholder"] = (
SimpleNamespace(prefill_backend=prefill_backend)
)
builder = backend_cls.get_builder_cls()(
kv_cache_spec, ["placeholder"], vllm_config, device
)
+9 -4
View File
@@ -1031,6 +1031,11 @@ def run_attention_backend(
@pytest.mark.parametrize("kv_cache_dtype", ["auto", "fp8", "fp8_e4m3"])
@pytest.mark.parametrize(("q_scale", "k_scale"), [(1.0, 1.0), (2.0, 3.0)])
@pytest.mark.parametrize("prefill_backend", PREFILL_BACKENDS_TO_TEST)
@pytest.mark.parametrize(
("qk_nope_head_dim", "v_head_dim"),
[(128, 128), (192, 256)],
ids=["deepseek", "glm"],
)
def test_backend_correctness(
default_vllm_config,
dist_init,
@@ -1042,6 +1047,8 @@ def test_backend_correctness(
q_scale: float,
k_scale: float,
prefill_backend: MLAPrefillBackendEnum,
qk_nope_head_dim: int,
v_head_dim: int,
):
"""
Test that all backends produce similar outputs to a reference implementation
@@ -1090,9 +1097,9 @@ def test_backend_correctness(
MLAPrefillSelectorConfig(
dtype=torch.bfloat16,
mla_dimensions=MLADimensions(
qk_nope_head_dim=128,
qk_nope_head_dim=qk_nope_head_dim,
qk_rope_head_dim=64,
v_head_dim=128,
v_head_dim=v_head_dim,
),
),
)
@@ -1168,8 +1175,6 @@ def test_backend_correctness(
dtype = _convert_dtype_to_torch(vllm_config.model_config.dtype)
kv_lora_rank = 512
qk_rope_head_dim = 64
qk_nope_head_dim = 128
v_head_dim = 128
total_head_size = kv_lora_rank + qk_rope_head_dim
assert kv_lora_rank + qk_rope_head_dim == head_size, (
f"MLA dimensions don't match: {total_head_size} != {head_size}"
@@ -442,6 +442,23 @@ def test_sparse_backend_decode_correctness(
scale=kv_cache_scale,
)
# The sparse builder clones the layer's dense-MHA prefill backend from
# static_forward_context; register a mock layer carrying one.
from vllm.v1.attention.backends.mla.prefill import get_mla_prefill_backend
prefill_backend = get_mla_prefill_backend(vllm_config)(
num_heads=num_heads,
scale=scale,
kv_lora_rank=kv_lora_rank,
qk_nope_head_dim=qk_nope_head_dim,
qk_rope_head_dim=qk_rope_head_dim,
v_head_dim=v_head_dim,
vllm_config=vllm_config,
)
vllm_config.compilation_config.static_forward_context["placeholder"] = (
SimpleNamespace(prefill_backend=prefill_backend)
)
builder_cls = backend_cls.get_builder_cls()
builder = builder_cls(kv_cache_spec, ["placeholder"], vllm_config, device)
metadata = builder.build(
@@ -722,6 +739,258 @@ def test_split_prefill_chunks(seq_lens, max_buf, expected):
assert out == expected
PREFILL_BATCH_SPECS = {
"short_dense_mha": BatchSpec(seq_lens=[64, 128], query_lens=[64, 128]),
"short_context_dense_mha": BatchSpec(seq_lens=[128, 160], query_lens=[64, 32]),
}
@pytest.mark.skipif(
torch.cuda.get_device_capability()[0] < 10,
reason="Sparse MLA forward_mha requires FA4 (SM100+)",
)
@pytest.mark.parametrize("batch_name", list(PREFILL_BATCH_SPECS.keys()))
@pytest.mark.parametrize("kv_cache_dtype", ["auto"])
def test_sparse_backend_prefill_correctness(
default_vllm_config,
dist_init,
batch_name,
kv_cache_dtype,
workspace_init,
):
"""Test single-pass FA4 dense forward_mha for sparse MLA prefill."""
backend_cls = FlashMLASparseBackend
batch_spec = PREFILL_BATCH_SPECS[batch_name]
device = torch.device("cuda")
dtype = torch.bfloat16
block_size = 64
num_heads = 128
kv_lora_rank = 512
qk_nope_head_dim = 128
qk_rope_head_dim = 64
v_head_dim = 128
head_size = kv_lora_rank + qk_rope_head_dim
topk_tokens = 512
max_seqlen = max(batch_spec.seq_lens)
total_cache_tokens = sum(batch_spec.seq_lens)
vllm_config = create_vllm_config(
model_name="deepseek-ai/DeepSeek-V2-Lite-Chat",
tensor_parallel_size=1,
max_model_len=max_seqlen,
num_gpu_blocks=max(2048, cdiv(total_cache_tokens, block_size) + 1),
block_size=block_size,
hf_config_override={
"index_topk": topk_tokens,
"attn_module_list_cfg": [{"topk_tokens": topk_tokens}],
},
)
model_config = vllm_config.model_config
model_config.hf_text_config = SimpleNamespace(
q_lora_rank=None,
kv_lora_rank=kv_lora_rank,
qk_nope_head_dim=qk_nope_head_dim,
qk_rope_head_dim=qk_rope_head_dim,
v_head_dim=v_head_dim,
model_type="deepseek_v2",
)
model_config.dtype = dtype
model_config.get_num_attention_heads = MethodType(
lambda self, parallel_config: num_heads, model_config
)
model_config.get_num_kv_heads = MethodType(
lambda self, parallel_config: 1, model_config
)
model_config.get_head_size = MethodType(lambda self: head_size, model_config)
model_config.get_sliding_window = MethodType(lambda self: None, model_config)
kv_cache_spec = create_standard_kv_cache_spec(vllm_config)
scale = 1.0 / math.sqrt(qk_nope_head_dim + qk_rope_head_dim)
torch.manual_seed(42)
W_UK = torch.rand(
kv_lora_rank, num_heads, qk_nope_head_dim, dtype=dtype, device=device
)
W_UV = torch.rand(kv_lora_rank, num_heads, v_head_dim, dtype=dtype, device=device)
seq_lens = batch_spec.seq_lens
query_lens = batch_spec.query_lens
# Compute dense reference outputs.
total_query_tokens = sum(query_lens)
sparse_indices = torch.zeros(
total_query_tokens, topk_tokens, dtype=torch.int32, device=device
)
all_q, all_kv_c_new, all_k_pe_new = [], [], []
kv_c_contexts, k_pe_contexts = [], []
reference_outputs = []
for i in range(batch_spec.batch_size):
s_len = seq_lens[i]
q_len = query_lens[i]
ctx_len = s_len - q_len
q_mha = torch.rand(
q_len,
num_heads,
qk_nope_head_dim + qk_rope_head_dim,
dtype=dtype,
device=device,
)
kv_c_full = torch.rand(s_len, kv_lora_rank, dtype=dtype, device=device)
k_pe_full = torch.rand(s_len, 1, qk_rope_head_dim, dtype=dtype, device=device)
# Decompress all KV for reference
kv_b_weight = torch.cat([W_UK, W_UV], dim=-1).view(
kv_lora_rank, num_heads * (qk_nope_head_dim + v_head_dim)
)
kv_decompressed = (kv_c_full @ kv_b_weight).view(
s_len, num_heads, qk_nope_head_dim + v_head_dim
)
k_nope_all, v_all = kv_decompressed.split(
[qk_nope_head_dim, v_head_dim], dim=-1
)
k_pe_expanded = k_pe_full.expand(-1, num_heads, -1)
k_all = torch.cat([k_nope_all, k_pe_expanded], dim=-1)
for j in range(q_len):
attend_end = ctx_len + j + 1
q_tok = q_mha[j : j + 1] # (1, H, D_qk)
k_attend = k_all[:attend_end] # (N, H, D_qk)
v_attend = v_all[:attend_end] # (N, H, D_v)
q_sdpa = q_tok.unsqueeze(0).transpose(1, 2).float()
k_sdpa = k_attend.unsqueeze(0).transpose(1, 2).float()
v_sdpa = v_attend.unsqueeze(0).transpose(1, 2).float()
out = torch.nn.functional.scaled_dot_product_attention(
q_sdpa, k_sdpa, v_sdpa, scale=scale
)
out = out.transpose(1, 2).squeeze(0) # (1, H, D_v)
reference_outputs.append(out.to(dtype).flatten(start_dim=-2))
all_q.append(q_mha)
all_kv_c_new.append(kv_c_full[ctx_len:])
all_k_pe_new.append(k_pe_full[ctx_len:])
kv_c_contexts.append(kv_c_full)
k_pe_contexts.append(k_pe_full)
query_cat = torch.cat(all_q, dim=0)
kv_c_cat = torch.cat(all_kv_c_new, dim=0)
k_pe_cat = torch.cat(all_k_pe_new, dim=0)
ref_output = torch.cat(reference_outputs, dim=0)
vllm_config.cache_config.cache_dtype = kv_cache_dtype
vllm_config.model_config.hf_config.index_topk = topk_tokens
common_attn_metadata = create_common_attn_metadata(
batch_spec,
vllm_config.cache_config.block_size,
device,
arange_block_indices=True,
)
kv_cache = create_and_prepopulate_kv_cache(
kv_c_contexts=kv_c_contexts,
k_pe_contexts=k_pe_contexts,
block_size=block_size,
head_size=head_size,
dtype=dtype,
device=device,
num_blocks=vllm_config.cache_config.num_gpu_blocks,
common_attn_metadata=common_attn_metadata,
randomize_blocks=False,
kv_cache_dtype=kv_cache_dtype,
)
# The sparse builder clones the layer's dense-MHA prefill backend from
# static_forward_context; register a mock layer carrying one.
from vllm.v1.attention.backends.mla.prefill import get_mla_prefill_backend
prefill_backend = get_mla_prefill_backend(vllm_config)(
num_heads=num_heads,
scale=scale,
kv_lora_rank=kv_lora_rank,
qk_nope_head_dim=qk_nope_head_dim,
qk_rope_head_dim=qk_rope_head_dim,
v_head_dim=v_head_dim,
vllm_config=vllm_config,
)
vllm_config.compilation_config.static_forward_context["placeholder"] = (
SimpleNamespace(prefill_backend=prefill_backend)
)
builder_cls = backend_cls.get_builder_cls()
builder = builder_cls(kv_cache_spec, ["placeholder"], vllm_config, device)
# Drive the queries through the dense-MHA prefill path directly (the routing
# threshold would otherwise classify these short queries as MQA decodes).
builder.reorder_batch_threshold = 1
metadata = builder.build(
common_prefix_len=0, common_attn_metadata=common_attn_metadata
)
mock_indexer = SimpleNamespace(topk_indices_buffer=sparse_indices)
kv_b_proj_weight = torch.cat([W_UK, W_UV], dim=-1).view(
kv_lora_rank, num_heads * (qk_nope_head_dim + v_head_dim)
)
mock_kv_b_proj = ColumnParallelLinear(
input_size=kv_lora_rank,
output_size=num_heads * (qk_nope_head_dim + v_head_dim),
bias=False,
).to(device=device, dtype=dtype)
mock_kv_b_proj.weight = torch.nn.Parameter(kv_b_proj_weight.T.contiguous())
impl_cls = backend_cls.get_impl_cls()
with set_current_vllm_config(vllm_config):
impl = impl_cls(
num_heads=num_heads,
head_size=head_size,
scale=scale,
num_kv_heads=1,
alibi_slopes=None,
sliding_window=None,
kv_cache_dtype=kv_cache_dtype,
logits_soft_cap=None,
attn_type="decoder",
kv_sharing_target_layer_name=None,
q_lora_rank=None,
kv_lora_rank=kv_lora_rank,
qk_nope_head_dim=qk_nope_head_dim,
qk_rope_head_dim=qk_rope_head_dim,
qk_head_dim=qk_nope_head_dim + qk_rope_head_dim,
v_head_dim=v_head_dim,
kv_b_proj=mock_kv_b_proj,
indexer=mock_indexer,
)
impl.process_weights_after_loading(dtype)
out_buffer = torch.empty(
total_query_tokens, num_heads * v_head_dim, dtype=dtype, device=device
)
with torch.inference_mode():
impl.forward_mha(
q=query_cat,
kv_c_normed=kv_c_cat,
k_pe=k_pe_cat,
kv_c_and_k_pe_cache=kv_cache,
attn_metadata=metadata,
k_scale=torch.tensor(1.0, device=device),
output=out_buffer,
)
assert out_buffer.shape == ref_output.shape
assert torch.isfinite(out_buffer).all(), "Non-finite values in output"
torch.testing.assert_close(out_buffer, ref_output, rtol=0.01, atol=0.01)
@pytest.mark.parametrize(
"seq_lens,query_lens,workspace_size,max_logits_bytes,expected",
[