[Kimi-K3] Use the fused K/V pack kernel on the decode-context-parallel prefill path (#56881)
Signed-off-by: Brandon Zhang <31413216+brandonfzhang@users.noreply.github.com> Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com> Co-authored-by: Wei Zhao <51183510+wzhao18@users.noreply.github.com>
This commit is contained in:
co-authored by
Claude Fable 5.1
Wei Zhao
parent
5cd8a2626b
commit
b2863c0de9
@@ -785,6 +785,15 @@ class MultiHeadLatentAttention(nn.Module, AttentionLayerBase):
|
||||
cos_sin_cache=cos_sin_cache,
|
||||
)
|
||||
|
||||
def _fused_mla_kv_concat(
|
||||
self, kv_nope: torch.Tensor, k_pe: torch.Tensor, use_fp8_prefill: bool
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Pack one DCP context chunk's ``(k, v)`` with the fused kernels."""
|
||||
k_nope, v = kv_nope.split([self.qk_nope_head_dim, self.v_head_dim], dim=-1)
|
||||
if use_fp8_prefill:
|
||||
return fused_mla_kv_concat_quant_fp8(k_nope, k_pe, v)
|
||||
return fused_mla_kv_concat(k_nope, k_pe), v
|
||||
|
||||
def _compute_prefill_context(
|
||||
self,
|
||||
q: torch.Tensor,
|
||||
@@ -814,8 +823,8 @@ class MultiHeadLatentAttention(nn.Module, AttentionLayerBase):
|
||||
copied per chunk.
|
||||
|
||||
Decode context parallelism keeps using
|
||||
``impl._context_parallel_compute_prefill_context``; its extra allgather
|
||||
and reorg are not fused here.
|
||||
``impl._context_parallel_compute_prefill_context`` for its allgather and
|
||||
reorg, with this layer's ``_fused_mla_kv_concat`` as its per-chunk pack.
|
||||
"""
|
||||
prefill = attn_metadata.prefill
|
||||
assert prefill is not None
|
||||
@@ -1074,6 +1083,7 @@ class MultiHeadLatentAttention(nn.Module, AttentionLayerBase):
|
||||
attn_metadata,
|
||||
k_scale=self._k_scale,
|
||||
dcp_world_size=self.dcp_world_size,
|
||||
fused_mla_kv_concat_fn=self._fused_mla_kv_concat,
|
||||
)
|
||||
)
|
||||
else:
|
||||
|
||||
Reference in New Issue
Block a user