[Perf][GLM-5.3-Flash] Decode hot-path cleanups: strided KDA recurrent inputs, NoPE MQA query without concat, no duplicate router GEMM (#55736)
Signed-off-by: Jared Wen <jaredwen@inferact.ai> Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Fable 5.1
parent
b47b01cf33
commit
1768273c13
@@ -998,14 +998,17 @@ class MLAAttention(nn.Module, AttentionLayerBase):
|
||||
if self.q_pad_num_heads is not None:
|
||||
mqa_ql_nope = mqa_q_nope.new_empty((self.q_pad_num_heads, B, L))
|
||||
mqa_ql_nope.resize_((N, B, L))
|
||||
# Multiply (N, B, P) x (N, P, L) -> (N, B, L)
|
||||
torch.bmm(mqa_q_nope, W_UK_T, out=mqa_ql_nope)
|
||||
# Convert from (N, B, L) to (B, N, L)
|
||||
mqa_ql_nope = mqa_ql_nope.transpose(0, 1)
|
||||
else:
|
||||
mqa_ql_nope = mqa_q_nope.new_empty((N, B, L))
|
||||
|
||||
# Multiply (N, B, P) x (N, P, L) -> (N, B, L)
|
||||
torch.bmm(mqa_q_nope, W_UK_T, out=mqa_ql_nope)
|
||||
|
||||
# Convert from (N, B, L) to (B, N, L)
|
||||
mqa_ql_nope = mqa_ql_nope.transpose(0, 1)
|
||||
# Write the (N, B, L) bmm result straight into a
|
||||
# token-major (B, N, L) buffer so the MQA query is already
|
||||
# contiguous; a NoPE model (qk_rope_head_dim == 0) then
|
||||
# needs no concat at all.
|
||||
mqa_ql_nope = mqa_q_nope.new_empty((B, N, L))
|
||||
torch.bmm(mqa_q_nope, W_UK_T, out=mqa_ql_nope.transpose(0, 1))
|
||||
|
||||
if fp8_attention and self.impl.supports_quant_query_input:
|
||||
assert mqa_ql_nope.shape[0] == mqa_q_pe.shape[0]
|
||||
|
||||
Reference in New Issue
Block a user