[Attention][MLA] Fuse Kimi-K3 chunked-context K/V packing (#51772)

Signed-off-by: Yongye Zhu <zyy1102000@gmail.com>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Yongye Zhu
2026-08-12 22:42:57 -07:00
committed by GitHub
co-authored by Claude Opus 5 mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 63344913a6
commit 903d2efe7e
15 changed files with 996 additions and 35 deletions
+11
View File
@@ -466,6 +466,13 @@ STABLE_TORCH_LIBRARY_FRAGMENT(_C, ops) {
"Tensor! k_out, Tensor! k_cache, Tensor slot_mapping, "
"int cache_block_size, Tensor? position_ids=None, "
"Tensor? cos_sin_cache=None) -> ()");
ops.def(
"fused_kimi_k3_mla_kv_concat(Tensor k_nope, Tensor k_pe, Tensor! k_out) "
"-> ()");
ops.def(
"fused_kimi_k3_mla_kv_concat_quant_fp8("
"Tensor k_nope, Tensor k_pe, Tensor v, Tensor! k_fp8, Tensor! v_fp8) "
"-> ()");
ops.def(
"fused_kimi_k3_mla_qkv_quant_kv_cache_fp8_insert("
"Tensor q, Tensor k_nope, Tensor k_pe, Tensor kv_c_normed, Tensor v, "
@@ -769,6 +776,10 @@ STABLE_TORCH_LIBRARY_IMPL(_C, CUDA, ops) {
TORCH_BOX(&fused_kimi_k3_mla_key_concat_kv_cache_insert));
ops.impl("fused_kimi_k3_mla_key_concat_ds_mla_insert",
TORCH_BOX(&fused_kimi_k3_mla_key_concat_ds_mla_insert));
ops.impl("fused_kimi_k3_mla_kv_concat",
TORCH_BOX(&fused_kimi_k3_mla_kv_concat));
ops.impl("fused_kimi_k3_mla_kv_concat_quant_fp8",
TORCH_BOX(&fused_kimi_k3_mla_kv_concat_quant_fp8));
ops.impl("fused_kimi_k3_mla_qkv_quant_kv_cache_fp8_insert",
TORCH_BOX(&fused_kimi_k3_mla_qkv_quant_kv_cache_fp8_insert));
ops.impl("fused_kimi_k3_mla_decode_q_concat_kv_cache_insert",