[Attention][MLA] Fuse Kimi-K3 chunked-context K/V packing (#51772)
Signed-off-by: Yongye Zhu <zyy1102000@gmail.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
Claude Opus 5
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
63344913a6
commit
903d2efe7e
@@ -466,6 +466,13 @@ STABLE_TORCH_LIBRARY_FRAGMENT(_C, ops) {
|
||||
"Tensor! k_out, Tensor! k_cache, Tensor slot_mapping, "
|
||||
"int cache_block_size, Tensor? position_ids=None, "
|
||||
"Tensor? cos_sin_cache=None) -> ()");
|
||||
ops.def(
|
||||
"fused_kimi_k3_mla_kv_concat(Tensor k_nope, Tensor k_pe, Tensor! k_out) "
|
||||
"-> ()");
|
||||
ops.def(
|
||||
"fused_kimi_k3_mla_kv_concat_quant_fp8("
|
||||
"Tensor k_nope, Tensor k_pe, Tensor v, Tensor! k_fp8, Tensor! v_fp8) "
|
||||
"-> ()");
|
||||
ops.def(
|
||||
"fused_kimi_k3_mla_qkv_quant_kv_cache_fp8_insert("
|
||||
"Tensor q, Tensor k_nope, Tensor k_pe, Tensor kv_c_normed, Tensor v, "
|
||||
@@ -769,6 +776,10 @@ STABLE_TORCH_LIBRARY_IMPL(_C, CUDA, ops) {
|
||||
TORCH_BOX(&fused_kimi_k3_mla_key_concat_kv_cache_insert));
|
||||
ops.impl("fused_kimi_k3_mla_key_concat_ds_mla_insert",
|
||||
TORCH_BOX(&fused_kimi_k3_mla_key_concat_ds_mla_insert));
|
||||
ops.impl("fused_kimi_k3_mla_kv_concat",
|
||||
TORCH_BOX(&fused_kimi_k3_mla_kv_concat));
|
||||
ops.impl("fused_kimi_k3_mla_kv_concat_quant_fp8",
|
||||
TORCH_BOX(&fused_kimi_k3_mla_kv_concat_quant_fp8));
|
||||
ops.impl("fused_kimi_k3_mla_qkv_quant_kv_cache_fp8_insert",
|
||||
TORCH_BOX(&fused_kimi_k3_mla_qkv_quant_kv_cache_fp8_insert));
|
||||
ops.impl("fused_kimi_k3_mla_decode_q_concat_kv_cache_insert",
|
||||
|
||||
Reference in New Issue
Block a user