[Perf][GLM-5.3-Flash] Dense/masked-MHA sparse prefill for the NoPE (256, 0, 256) layout + skip the NoPE K concat (#55738)
Signed-off-by: Jared Wen <jaredwen@inferact.ai> Signed-off-by: Jared Wen <w13431838023@gmail.com> Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
Claude Fable 5.1
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
79f0be21ff
commit
238cb2b191
@@ -2871,6 +2871,10 @@ class MLACommonBaseImpl(MLAAttentionImpl[A], Generic[A]):
|
||||
Returns:
|
||||
Tensor of shape [..., nope_dim + pe_dim]
|
||||
"""
|
||||
if k_pe.shape[-1] == 0:
|
||||
# NoPE MLA: nothing to append, so no copy either.
|
||||
return k_nope
|
||||
|
||||
k = torch.empty(
|
||||
(*k_nope.shape[:-1], k_nope.shape[-1] + k_pe.shape[-1]),
|
||||
dtype=k_nope.dtype,
|
||||
|
||||
@@ -111,6 +111,8 @@ def _is_masked_mha_available(
|
||||
if model_dims not in (
|
||||
(128, 512, 128, 64, 128),
|
||||
(64, 512, 192, 64, 256),
|
||||
# GLM-5.3-Flash: NoPE, qk_head_dim 256 == the (192, 64, 256) kernel.
|
||||
(64, 512, 256, 0, 256),
|
||||
):
|
||||
return False
|
||||
qk_head_dim = qk_nope_head_dim + qk_rope_head_dim
|
||||
|
||||
Reference in New Issue
Block a user