[Kimi-K3] Use the fused K/V pack kernel on the decode-context-parallel prefill path (#56881)

Signed-off-by: Brandon Zhang <31413216+brandonfzhang@users.noreply.github.com>
Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
Co-authored-by: Wei Zhao <51183510+wzhao18@users.noreply.github.com>
This commit is contained in:
Brandon Zhang
2026-10-06 00:13:55 +00:00
committed by GitHub
co-authored by Claude Fable 5.1 Wei Zhao
parent 5cd8a2626b
commit b2863c0de9
3 changed files with 142 additions and 7 deletions
+12 -2
View File
@@ -785,6 +785,15 @@ class MultiHeadLatentAttention(nn.Module, AttentionLayerBase):
cos_sin_cache=cos_sin_cache,
)
def _fused_mla_kv_concat(
self, kv_nope: torch.Tensor, k_pe: torch.Tensor, use_fp8_prefill: bool
) -> tuple[torch.Tensor, torch.Tensor]:
"""Pack one DCP context chunk's ``(k, v)`` with the fused kernels."""
k_nope, v = kv_nope.split([self.qk_nope_head_dim, self.v_head_dim], dim=-1)
if use_fp8_prefill:
return fused_mla_kv_concat_quant_fp8(k_nope, k_pe, v)
return fused_mla_kv_concat(k_nope, k_pe), v
def _compute_prefill_context(
self,
q: torch.Tensor,
@@ -814,8 +823,8 @@ class MultiHeadLatentAttention(nn.Module, AttentionLayerBase):
copied per chunk.
Decode context parallelism keeps using
``impl._context_parallel_compute_prefill_context``; its extra allgather
and reorg are not fused here.
``impl._context_parallel_compute_prefill_context`` for its allgather and
reorg, with this layer's ``_fused_mla_kv_concat`` as its per-chunk pack.
"""
prefill = attn_metadata.prefill
assert prefill is not None
@@ -1074,6 +1083,7 @@ class MultiHeadLatentAttention(nn.Module, AttentionLayerBase):
attn_metadata,
k_scale=self._k_scale,
dcp_world_size=self.dcp_world_size,
fused_mla_kv_concat_fn=self._fused_mla_kv_concat,
)
)
else: