[Perf][GLM5.2] Fuse sparse MLA Q concatenation with head padding (#53878)
Signed-off-by: chaunceyjiang <chaunceyjiang@gmail.com>
This commit is contained in:
@@ -34,6 +34,17 @@ def test_concat_mla_q_contiguous(num_tokens, num_heads, nope_dim, rope_dim, dtyp
|
||||
torch.testing.assert_close(q_out, ref, atol=0, rtol=0)
|
||||
|
||||
|
||||
def test_concat_mla_q_padded_output_heads():
|
||||
"""The fused concat can write directly into a head-padded workspace."""
|
||||
ql_nope = torch.randn(4, 8, 512, dtype=torch.bfloat16, device="cuda")
|
||||
q_pe = torch.randn(4, 8, 64, dtype=torch.bfloat16, device="cuda")
|
||||
q_out = torch.empty(4, 64, 576, dtype=torch.bfloat16, device="cuda")
|
||||
|
||||
ops.concat_mla_q(ql_nope, q_pe, q_out)
|
||||
|
||||
torch.testing.assert_close(q_out[:, :8], torch.cat((ql_nope, q_pe), dim=-1))
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_tokens", [t for t in NUM_TOKENS if t > 1])
|
||||
@pytest.mark.parametrize("num_heads", NUM_HEADS)
|
||||
@pytest.mark.parametrize("nope_dim", NOPE_DIM)
|
||||
|
||||
@@ -824,18 +824,20 @@ def test_flashmla_forward_bf16_kv_slices_req_id_to_mqa_tokens():
|
||||
|
||||
captured = {}
|
||||
|
||||
def _stub_kernel(q, kv, indices, lengths):
|
||||
def _stub_kernel(q, kv, indices, lengths, actual_num_heads):
|
||||
captured["indices"] = indices
|
||||
captured["actual_num_heads"] = actual_num_heads
|
||||
return torch.zeros(q.shape[0], q.shape[1], 512, dtype=q.dtype, device=q.device)
|
||||
|
||||
stub_impl = SimpleNamespace(_bf16_flash_mla_kernel=_stub_kernel)
|
||||
|
||||
out = FlashMLASparseImpl._forward_bf16_kv(
|
||||
stub_impl, q, kv_cache, topk_indices, attn_metadata
|
||||
stub_impl, q, kv_cache, topk_indices, attn_metadata, q.shape[1]
|
||||
)
|
||||
|
||||
assert out.shape[0] == num_mqa_tokens
|
||||
assert captured["indices"].shape[0] == num_mqa_tokens
|
||||
assert captured["actual_num_heads"] == q.shape[1]
|
||||
reference = _triton_convert_reference_impl(
|
||||
attn_metadata.req_id_per_token[:num_mqa_tokens],
|
||||
attn_metadata.block_table,
|
||||
|
||||
@@ -587,7 +587,14 @@ class FlashMLASparseImpl(SparseMLACommonImpl[FlashMLASparseMetadata]):
|
||||
|
||||
vllm_config = get_current_vllm_config()
|
||||
max_tokens = vllm_config.scheduler_config.max_num_batched_tokens
|
||||
q_concat_shape = (max_tokens, num_heads, head_size)
|
||||
q_concat_heads = num_heads
|
||||
if not is_quantized_kv_cache(kv_cache_dtype):
|
||||
q_concat_heads = (
|
||||
(num_heads + self.prefill_padding - 1)
|
||||
// self.prefill_padding
|
||||
* self.prefill_padding
|
||||
)
|
||||
q_concat_shape = (max_tokens, q_concat_heads, head_size)
|
||||
if is_quantized_kv_cache(kv_cache_dtype):
|
||||
assert kv_cache_dtype == "fp8_ds_mla", (
|
||||
"FlashMLA Sparse Attention backend fp8 only supports "
|
||||
@@ -626,6 +633,7 @@ class FlashMLASparseImpl(SparseMLACommonImpl[FlashMLASparseMetadata]):
|
||||
kv_c_and_k_pe_cache: torch.Tensor,
|
||||
topk_indices: torch.Tensor,
|
||||
attn_metadata: FlashMLASparseMetadata,
|
||||
actual_num_heads: int,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
# Convert per-request indices to global slots (decode) or workspace
|
||||
# offsets (prefill). req_id_per_token covers the whole batch; slice it
|
||||
@@ -648,6 +656,7 @@ class FlashMLASparseImpl(SparseMLACommonImpl[FlashMLASparseMetadata]):
|
||||
kv_rows,
|
||||
topk_indices,
|
||||
topk_length,
|
||||
actual_num_heads,
|
||||
)
|
||||
|
||||
def _forward_fp8_kv_separate_prefill_decode(
|
||||
@@ -880,6 +889,7 @@ class FlashMLASparseImpl(SparseMLACommonImpl[FlashMLASparseMetadata]):
|
||||
kv_c_and_k_pe_cache: torch.Tensor,
|
||||
topk_indices: torch.Tensor,
|
||||
topk_length: torch.Tensor | None = None,
|
||||
actual_num_heads: int | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
num_tokens = q.shape[0]
|
||||
kv_c_and_k_pe_cache = kv_c_and_k_pe_cache.view(
|
||||
@@ -889,13 +899,14 @@ class FlashMLASparseImpl(SparseMLACommonImpl[FlashMLASparseMetadata]):
|
||||
# NOTE(Chen): kernel requires num_local_head to be a multiple of
|
||||
# 64 on hopper and 128 on blackwell. Pad from q's head count, not
|
||||
# self.num_heads: under DCP the heads are all-gathered before this.
|
||||
actual_num_heads = q.shape[1]
|
||||
if actual_num_heads is None:
|
||||
actual_num_heads = q.shape[1]
|
||||
padded_num_heads = (
|
||||
(actual_num_heads + self.prefill_padding - 1)
|
||||
// self.prefill_padding
|
||||
* self.prefill_padding
|
||||
)
|
||||
if actual_num_heads < padded_num_heads:
|
||||
if q.shape[1] < padded_num_heads:
|
||||
logger.warning_once(
|
||||
f"Padding num_heads from {actual_num_heads} to "
|
||||
f"{padded_num_heads} for BF16 sparse prefill kernel"
|
||||
@@ -928,10 +939,13 @@ class FlashMLASparseImpl(SparseMLACommonImpl[FlashMLASparseMetadata]):
|
||||
# MQA 576/512 approach for both prefill and decode
|
||||
|
||||
# Concatenate q if it's a tuple (ql_nope, q_pe)
|
||||
actual_num_heads = self.num_heads
|
||||
if isinstance(q, tuple):
|
||||
ql_nope, q_pe = q
|
||||
q = self.q_concat_buffer[: ql_nope.shape[0]]
|
||||
ops.concat_mla_q(ql_nope, q_pe, q)
|
||||
else:
|
||||
actual_num_heads = q.shape[1]
|
||||
|
||||
num_actual_toks = q.shape[0]
|
||||
|
||||
@@ -945,7 +959,11 @@ class FlashMLASparseImpl(SparseMLACommonImpl[FlashMLASparseMetadata]):
|
||||
|
||||
if not use_fp8_cache:
|
||||
attn_out, bf16_lse = self._forward_bf16_kv(
|
||||
q, kv_c_and_k_pe_cache, topk_indices, attn_metadata
|
||||
q,
|
||||
kv_c_and_k_pe_cache,
|
||||
topk_indices,
|
||||
attn_metadata,
|
||||
actual_num_heads,
|
||||
)
|
||||
if self.need_to_return_lse_for_decode:
|
||||
lse = bf16_lse
|
||||
|
||||
Reference in New Issue
Block a user