[Perf][GLM-5.3-Flash] Decode hot-path cleanups: strided KDA recurrent inputs, NoPE MQA query without concat, no duplicate router GEMM (#55736)

Signed-off-by: Jared Wen <jaredwen@inferact.ai>
Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
Jared Wen
2026-09-10 16:28:33 +08:00
committed by GitHub
co-authored by Claude Fable 5.1
parent b47b01cf33
commit 1768273c13
8 changed files with 268 additions and 27 deletions
@@ -998,14 +998,17 @@ class MLAAttention(nn.Module, AttentionLayerBase):
if self.q_pad_num_heads is not None:
mqa_ql_nope = mqa_q_nope.new_empty((self.q_pad_num_heads, B, L))
mqa_ql_nope.resize_((N, B, L))
# Multiply (N, B, P) x (N, P, L) -> (N, B, L)
torch.bmm(mqa_q_nope, W_UK_T, out=mqa_ql_nope)
# Convert from (N, B, L) to (B, N, L)
mqa_ql_nope = mqa_ql_nope.transpose(0, 1)
else:
mqa_ql_nope = mqa_q_nope.new_empty((N, B, L))
# Multiply (N, B, P) x (N, P, L) -> (N, B, L)
torch.bmm(mqa_q_nope, W_UK_T, out=mqa_ql_nope)
# Convert from (N, B, L) to (B, N, L)
mqa_ql_nope = mqa_ql_nope.transpose(0, 1)
# Write the (N, B, L) bmm result straight into a
# token-major (B, N, L) buffer so the MQA query is already
# contiguous; a NoPE model (qk_rope_head_dim == 0) then
# needs no concat at all.
mqa_ql_nope = mqa_q_nope.new_empty((B, N, L))
torch.bmm(mqa_q_nope, W_UK_T, out=mqa_ql_nope.transpose(0, 1))
if fp8_attention and self.impl.supports_quant_query_input:
assert mqa_ql_nope.shape[0] == mqa_q_pe.shape[0]