[Kimi-K3] Use the fused K/V pack kernel on the decode-context-parallel prefill path (#56881)
Signed-off-by: Brandon Zhang <31413216+brandonfzhang@users.noreply.github.com> Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com> Co-authored-by: Wei Zhao <51183510+wzhao18@users.noreply.github.com>
This commit is contained in:
co-authored by
Claude Fable 5.1
Wei Zhao
parent
5cd8a2626b
commit
b2863c0de9
@@ -3033,7 +3033,13 @@ class MLACommonBaseImpl(MLAAttentionImpl[A], Generic[A]):
|
||||
attn_metadata: MLACommonMetadata,
|
||||
k_scale: torch.Tensor,
|
||||
dcp_world_size: int,
|
||||
fused_mla_kv_concat_fn: Callable[
|
||||
[torch.Tensor, torch.Tensor, bool], tuple[torch.Tensor, torch.Tensor]
|
||||
]
|
||||
| None = None,
|
||||
):
|
||||
# fused_mla_kv_concat_fn(kv_nope, k_pe, use_fp8_prefill) -> (k, v) replaces
|
||||
# the per-chunk cast + split + concat below with one fused kernel.
|
||||
assert attn_metadata.prefill is not None
|
||||
prefill_metadata = attn_metadata.prefill
|
||||
assert prefill_metadata.prefill_backend is not None
|
||||
@@ -3127,11 +3133,16 @@ class MLACommonBaseImpl(MLAAttentionImpl[A], Generic[A]):
|
||||
kv_nope = self.kv_b_proj(kv_c_normed)[0].view(
|
||||
-1, self.num_heads, self.qk_nope_head_dim + self.v_head_dim
|
||||
)
|
||||
if use_fp8_prefill:
|
||||
kv_nope = kv_nope.to(prefill_metadata.q_data_type)
|
||||
k_pe = k_pe.to(prefill_metadata.q_data_type)
|
||||
k_nope, v = kv_nope.split([self.qk_nope_head_dim, self.v_head_dim], dim=-1)
|
||||
k = self._concat_k_nope_k_pe(k_nope, k_pe)
|
||||
if fused_mla_kv_concat_fn is not None:
|
||||
k, v = fused_mla_kv_concat_fn(kv_nope, k_pe, use_fp8_prefill)
|
||||
else:
|
||||
if use_fp8_prefill:
|
||||
kv_nope = kv_nope.to(prefill_metadata.q_data_type)
|
||||
k_pe = k_pe.to(prefill_metadata.q_data_type)
|
||||
k_nope, v = kv_nope.split(
|
||||
[self.qk_nope_head_dim, self.v_head_dim], dim=-1
|
||||
)
|
||||
k = self._concat_k_nope_k_pe(k_nope, k_pe)
|
||||
|
||||
attn_output, attn_softmax_lse = (
|
||||
prefill_metadata.prefill_backend.run_prefill_context_chunk(
|
||||
|
||||
Reference in New Issue
Block a user