[Kimi-K3] Use the fused K/V pack kernel on the decode-context-parallel prefill path (#56881)

Signed-off-by: Brandon Zhang <31413216+brandonfzhang@users.noreply.github.com>
Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
Co-authored-by: Wei Zhao <51183510+wzhao18@users.noreply.github.com>
This commit is contained in:
Brandon Zhang
2026-10-06 00:13:55 +00:00
committed by GitHub
co-authored by Claude Fable 5.1 Wei Zhao
parent 5cd8a2626b
commit b2863c0de9
3 changed files with 142 additions and 7 deletions
@@ -3033,7 +3033,13 @@ class MLACommonBaseImpl(MLAAttentionImpl[A], Generic[A]):
attn_metadata: MLACommonMetadata,
k_scale: torch.Tensor,
dcp_world_size: int,
fused_mla_kv_concat_fn: Callable[
[torch.Tensor, torch.Tensor, bool], tuple[torch.Tensor, torch.Tensor]
]
| None = None,
):
# fused_mla_kv_concat_fn(kv_nope, k_pe, use_fp8_prefill) -> (k, v) replaces
# the per-chunk cast + split + concat below with one fused kernel.
assert attn_metadata.prefill is not None
prefill_metadata = attn_metadata.prefill
assert prefill_metadata.prefill_backend is not None
@@ -3127,11 +3133,16 @@ class MLACommonBaseImpl(MLAAttentionImpl[A], Generic[A]):
kv_nope = self.kv_b_proj(kv_c_normed)[0].view(
-1, self.num_heads, self.qk_nope_head_dim + self.v_head_dim
)
if use_fp8_prefill:
kv_nope = kv_nope.to(prefill_metadata.q_data_type)
k_pe = k_pe.to(prefill_metadata.q_data_type)
k_nope, v = kv_nope.split([self.qk_nope_head_dim, self.v_head_dim], dim=-1)
k = self._concat_k_nope_k_pe(k_nope, k_pe)
if fused_mla_kv_concat_fn is not None:
k, v = fused_mla_kv_concat_fn(kv_nope, k_pe, use_fp8_prefill)
else:
if use_fp8_prefill:
kv_nope = kv_nope.to(prefill_metadata.q_data_type)
k_pe = k_pe.to(prefill_metadata.q_data_type)
k_nope, v = kv_nope.split(
[self.qk_nope_head_dim, self.v_head_dim], dim=-1
)
k = self._concat_k_nope_k_pe(k_nope, k_pe)
attn_output, attn_softmax_lse = (
prefill_metadata.prefill_backend.run_prefill_context_chunk(