[Perf][Attention] Reduce GLM sparse MLA preparation overhead (#57458)
Signed-off-by: Summer Yang <girasoleyang@gmail.com> Co-authored-by: Codex <noreply@openai.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: Jiangyun Zhu <riverclouds.zhu@qq.com>
This commit is contained in:
co-authored by
Codex
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Jiangyun Zhu
parent
c64b15cde5
commit
a369a7becc
@@ -1553,7 +1553,10 @@ class _DecodeConcatQuantFP8(QuantFP8):
|
||||
scale: torch.Tensor,
|
||||
scale_ub: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
decode_q0 = torch.cat((decode_ql_nope, decode_q_pe), dim=-1)
|
||||
if decode_q_pe.shape[-1] == 0 and decode_ql_nope.is_contiguous():
|
||||
decode_q0 = decode_ql_nope
|
||||
else:
|
||||
decode_q0 = torch.cat((decode_ql_nope, decode_q_pe), dim=-1)
|
||||
decode_q_flat = decode_q0.reshape(decode_q0.shape[0], -1)
|
||||
decode_q, _ = quant_fn(self, decode_q_flat, scale, scale_ub)
|
||||
return decode_q.view(decode_q0.shape)
|
||||
|
||||
Reference in New Issue
Block a user