[Perf][GLM5.2] Fuse sparse MLA Q concatenation with head padding (#53878)
Signed-off-by: chaunceyjiang <chaunceyjiang@gmail.com>
This commit is contained in:
@@ -34,6 +34,17 @@ def test_concat_mla_q_contiguous(num_tokens, num_heads, nope_dim, rope_dim, dtyp
|
||||
torch.testing.assert_close(q_out, ref, atol=0, rtol=0)
|
||||
|
||||
|
||||
def test_concat_mla_q_padded_output_heads():
|
||||
"""The fused concat can write directly into a head-padded workspace."""
|
||||
ql_nope = torch.randn(4, 8, 512, dtype=torch.bfloat16, device="cuda")
|
||||
q_pe = torch.randn(4, 8, 64, dtype=torch.bfloat16, device="cuda")
|
||||
q_out = torch.empty(4, 64, 576, dtype=torch.bfloat16, device="cuda")
|
||||
|
||||
ops.concat_mla_q(ql_nope, q_pe, q_out)
|
||||
|
||||
torch.testing.assert_close(q_out[:, :8], torch.cat((ql_nope, q_pe), dim=-1))
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_tokens", [t for t in NUM_TOKENS if t > 1])
|
||||
@pytest.mark.parametrize("num_heads", NUM_HEADS)
|
||||
@pytest.mark.parametrize("nope_dim", NOPE_DIM)
|
||||
|
||||
@@ -824,18 +824,20 @@ def test_flashmla_forward_bf16_kv_slices_req_id_to_mqa_tokens():
|
||||
|
||||
captured = {}
|
||||
|
||||
def _stub_kernel(q, kv, indices, lengths):
|
||||
def _stub_kernel(q, kv, indices, lengths, actual_num_heads):
|
||||
captured["indices"] = indices
|
||||
captured["actual_num_heads"] = actual_num_heads
|
||||
return torch.zeros(q.shape[0], q.shape[1], 512, dtype=q.dtype, device=q.device)
|
||||
|
||||
stub_impl = SimpleNamespace(_bf16_flash_mla_kernel=_stub_kernel)
|
||||
|
||||
out = FlashMLASparseImpl._forward_bf16_kv(
|
||||
stub_impl, q, kv_cache, topk_indices, attn_metadata
|
||||
stub_impl, q, kv_cache, topk_indices, attn_metadata, q.shape[1]
|
||||
)
|
||||
|
||||
assert out.shape[0] == num_mqa_tokens
|
||||
assert captured["indices"].shape[0] == num_mqa_tokens
|
||||
assert captured["actual_num_heads"] == q.shape[1]
|
||||
reference = _triton_convert_reference_impl(
|
||||
attn_metadata.req_id_per_token[:num_mqa_tokens],
|
||||
attn_metadata.block_table,
|
||||
|
||||
Reference in New Issue
Block a user