[Model] Add Kimi K3 support: model files and kernels [1/N] (#50089)
Signed-off-by: zjy0516 <riverclouds.zhu@qq.com> Signed-off-by: Isotr0py <Isotr0py@outlook.com> Signed-off-by: Bugen Zhao <i@bugenzhao.com> Signed-off-by: Yongye Zhu <zyy1102000@gmail.com> Co-authored-by: Isotr0py <Isotr0py@outlook.com> Co-authored-by: Thien Tran <gau.nernst@yahoo.com.sg> Co-authored-by: Bugen Zhao <i@bugenzhao.com> Co-authored-by: Yongye Zhu <zyy1102000@gmail.com> Co-authored-by: Ziming Huang <zelda.huanghuang@gmail.com> Co-authored-by: Roger Wang <hey@rogerw.io> Co-authored-by: Isotr0py <mozf@inferact.ai> Co-authored-by: aoshen02 <aoshen02@users.noreply.github.com> Co-authored-by: Woosuk Kwon <woosuk@inferact.ai> Co-authored-by: Jee Jee Li <jeejeelee@inferact.ai> Co-authored-by: aoshen02 <aoshen@inferact.ai> Co-authored-by: Summer Yang <girasoleyang@gmail.com> Co-authored-by: Kevin H. Luu <khluu000@gmail.com> Co-authored-by: Bowen Wang <abmfy@icloud.com> Co-authored-by: gnovack <novackgm@gmail.com> Co-authored-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai> Co-authored-by: xiaozhoupy <peiyuanzhou1994@gmail.com> Co-authored-by: Roy Wang <yasong.wang@inferact.ai> Co-authored-by: Jeff (Junze) Ma <93145857+majunze2001@users.noreply.github.com> Co-authored-by: Codex <codex@openai.com>
This commit is contained in:
co-authored by
Isotr0py
Thien Tran
Bugen Zhao
Yongye Zhu
Ziming Huang
Roger Wang
Isotr0py
aoshen02
Woosuk Kwon
Jee Jee Li
aoshen02
Summer Yang
Kevin H. Luu
Bowen Wang
gnovack
Nick Hill
Yifan Qiao
xiaozhoupy
Roy Wang
Jeff Ma
Codex
parent
6f00a1ae3b
commit
7c6729b769
@@ -324,7 +324,8 @@ STABLE_TORCH_LIBRARY_FRAGMENT(_C, ops) {
|
||||
// DeepSeek V3 fused A GEMM (SM 9.0+, bf16 only, 1-16 tokens).
|
||||
// conditionally compiled so impl registration is in source file
|
||||
ops.def(
|
||||
"dsv3_fused_a_gemm(Tensor! output, Tensor mat_a, Tensor mat_b) -> ()");
|
||||
"dsv3_fused_a_gemm(Tensor! output, Tensor mat_a, Tensor mat_b, "
|
||||
"bool enable_pdl=False) -> ()");
|
||||
|
||||
// BF16/FP32 x FP32 -> FP32 router GEMM for H=3072, E=256, M<=32 (SM90+).
|
||||
// conditionally compiled so impl registration is in source file
|
||||
@@ -447,6 +448,48 @@ STABLE_TORCH_LIBRARY_FRAGMENT(_C, ops) {
|
||||
"Tensor fp8_scale, Tensor q_fp8_scale_inv, float eps, "
|
||||
"int cache_block_size) -> ()");
|
||||
|
||||
// Kimi-K3 MLA epilogues: optional RoPE followed by concat/cache insertion.
|
||||
ops.def(
|
||||
"fused_kimi_k3_mla_key_concat_kv_cache_insert("
|
||||
"Tensor! q, Tensor k_nope, Tensor k_pe, Tensor kv_c_normed, "
|
||||
"Tensor! k_out, Tensor! k_cache, Tensor slot_mapping, "
|
||||
"int cache_block_size, Tensor? position_ids=None, "
|
||||
"Tensor? cos_sin_cache=None) -> ()");
|
||||
ops.def(
|
||||
"fused_kimi_k3_mla_key_concat_ds_mla_insert("
|
||||
"Tensor! q, Tensor k_nope, Tensor k_pe, Tensor kv_c_normed, "
|
||||
"Tensor! k_out, Tensor! k_cache, Tensor slot_mapping, "
|
||||
"int cache_block_size, Tensor? position_ids=None, "
|
||||
"Tensor? cos_sin_cache=None) -> ()");
|
||||
ops.def(
|
||||
"fused_kimi_k3_mla_qkv_quant_kv_cache_fp8_insert("
|
||||
"Tensor q, Tensor k_nope, Tensor k_pe, Tensor kv_c_normed, Tensor v, "
|
||||
"Tensor! q_fp8, Tensor! k_fp8, Tensor! v_fp8, Tensor! k_cache, "
|
||||
"Tensor slot_mapping, Tensor q_scale_inv, Tensor k_scale_inv, "
|
||||
"Tensor v_scale_inv, Tensor cache_scale_inv, int cache_block_size, "
|
||||
"Tensor? position_ids=None, Tensor? cos_sin_cache=None) -> ()");
|
||||
|
||||
// Kimi-K3 MLA decode epilogue: concat mqa_q = [ql_nope | q_pe] and insert the
|
||||
// latent [kv_c_normed | k_pe] into the paged cache (bf16 / fp8 / fp8_ds_mla).
|
||||
ops.def(
|
||||
"fused_kimi_k3_mla_decode_q_concat_kv_cache_insert("
|
||||
"Tensor ql_nope, Tensor q_pe, Tensor kv_c_normed, Tensor k_pe, "
|
||||
"Tensor! mqa_q, Tensor! k_cache, Tensor slot_mapping, "
|
||||
"int cache_block_size, Tensor? position_ids=None, "
|
||||
"Tensor? cos_sin_cache=None) -> ()");
|
||||
ops.def(
|
||||
"fused_kimi_k3_mla_decode_q_concat_kv_cache_fp8_insert("
|
||||
"Tensor ql_nope, Tensor q_pe, Tensor kv_c_normed, Tensor k_pe, "
|
||||
"Tensor! mqa_q, Tensor! k_cache, Tensor slot_mapping, "
|
||||
"Tensor q_scale_inv, Tensor cache_scale_inv, int cache_block_size, "
|
||||
"Tensor? position_ids=None, Tensor? cos_sin_cache=None) -> ()");
|
||||
ops.def(
|
||||
"fused_kimi_k3_mla_decode_q_concat_ds_mla_insert("
|
||||
"Tensor ql_nope, Tensor q_pe, Tensor kv_c_normed, Tensor k_pe, "
|
||||
"Tensor! mqa_q, Tensor! k_cache, Tensor slot_mapping, "
|
||||
"int cache_block_size, Tensor? position_ids=None, "
|
||||
"Tensor? cos_sin_cache=None) -> ()");
|
||||
|
||||
#ifndef USE_ROCM
|
||||
ops.def(
|
||||
"minimax_allreduce_rms_qk("
|
||||
@@ -468,6 +511,16 @@ STABLE_TORCH_LIBRARY_FRAGMENT(_C, ops) {
|
||||
"int block_size, Tensor!? q_out, Tensor!? index_q_out, "
|
||||
"str kv_cache_dtype, bool skip_index_branch=False) -> ()");
|
||||
|
||||
#ifdef VLLM_ENABLE_FUSED_KDA_DECODE
|
||||
ops.def(
|
||||
"fused_kda_decode("
|
||||
"Tensor x, Tensor weight, Tensor? bias, Tensor! conv_state, "
|
||||
"Tensor raw_g, Tensor raw_beta, Tensor A_log, Tensor dt_bias, "
|
||||
"Tensor state_indices, Tensor! state, Tensor! out, "
|
||||
"float? lower_bound=None, Tensor? output_gate=None, "
|
||||
"Tensor? norm_weight=None, float norm_eps=1e-5) -> ()");
|
||||
#endif
|
||||
|
||||
#ifdef VLLM_ENABLE_KIMI_K3_ATTN_RES
|
||||
ops.def(
|
||||
"kimi_k3_attn_res("
|
||||
@@ -538,6 +591,14 @@ STABLE_TORCH_LIBRARY_FRAGMENT(_C, ops) {
|
||||
"limit=7.0) "
|
||||
"-> ()");
|
||||
|
||||
// SituGLU implementation used in Kimi models.
|
||||
ops.def(
|
||||
"situ_and_mul(Tensor! out, Tensor input, float beta=1.0, float "
|
||||
"linear_beta=-1.0) -> ()");
|
||||
ops.def(
|
||||
"masked_situ_and_mul(Tensor! out, Tensor input, Tensor "
|
||||
"expert_num_tokens, float beta=1.0, float linear_beta=-1.0) -> ()");
|
||||
|
||||
// GELU implementation used in GPT-2.
|
||||
ops.def("gelu_new(Tensor! out, Tensor input) -> ()");
|
||||
|
||||
@@ -696,11 +757,27 @@ STABLE_TORCH_LIBRARY_IMPL(_C, CUDA, ops) {
|
||||
ops.impl(
|
||||
"fused_deepseek_v4_qnorm_rope_kv_rope_full_cache_fp8_insert",
|
||||
TORCH_BOX(&fused_deepseek_v4_qnorm_rope_kv_rope_full_cache_fp8_insert));
|
||||
ops.impl("fused_kimi_k3_mla_key_concat_kv_cache_insert",
|
||||
TORCH_BOX(&fused_kimi_k3_mla_key_concat_kv_cache_insert));
|
||||
ops.impl("fused_kimi_k3_mla_key_concat_ds_mla_insert",
|
||||
TORCH_BOX(&fused_kimi_k3_mla_key_concat_ds_mla_insert));
|
||||
ops.impl("fused_kimi_k3_mla_qkv_quant_kv_cache_fp8_insert",
|
||||
TORCH_BOX(&fused_kimi_k3_mla_qkv_quant_kv_cache_fp8_insert));
|
||||
ops.impl("fused_kimi_k3_mla_decode_q_concat_kv_cache_insert",
|
||||
TORCH_BOX(&fused_kimi_k3_mla_decode_q_concat_kv_cache_insert));
|
||||
ops.impl("fused_kimi_k3_mla_decode_q_concat_kv_cache_fp8_insert",
|
||||
TORCH_BOX(&fused_kimi_k3_mla_decode_q_concat_kv_cache_fp8_insert));
|
||||
ops.impl("fused_kimi_k3_mla_decode_q_concat_ds_mla_insert",
|
||||
TORCH_BOX(&fused_kimi_k3_mla_decode_q_concat_ds_mla_insert));
|
||||
#ifndef USE_ROCM
|
||||
ops.impl("minimax_allreduce_rms_qk", TORCH_BOX(&minimax_allreduce_rms_qk));
|
||||
#endif
|
||||
ops.impl("fused_minimax_m3_qknorm_rope_kv_insert",
|
||||
TORCH_BOX(&fused_minimax_m3_qknorm_rope_kv_insert));
|
||||
#ifdef VLLM_ENABLE_FUSED_KDA_DECODE
|
||||
ops.impl("fused_kda_decode", TORCH_BOX(&fused_kda_decode));
|
||||
#endif
|
||||
|
||||
#ifdef VLLM_ENABLE_KIMI_K3_ATTN_RES
|
||||
ops.impl("kimi_k3_attn_res", TORCH_BOX(&kimi_k3_attn_res));
|
||||
#endif
|
||||
@@ -726,6 +803,8 @@ STABLE_TORCH_LIBRARY_IMPL(_C, CUDA, ops) {
|
||||
ops.impl("gelu_tanh_and_mul", TORCH_BOX(&gelu_tanh_and_mul));
|
||||
ops.impl("fatrelu_and_mul", TORCH_BOX(&fatrelu_and_mul));
|
||||
ops.impl("swigluoai_and_mul", TORCH_BOX(&swigluoai_and_mul));
|
||||
ops.impl("situ_and_mul", TORCH_BOX(&situ_and_mul));
|
||||
ops.impl("masked_situ_and_mul", TORCH_BOX(&masked_situ_and_mul));
|
||||
ops.impl("gelu_new", TORCH_BOX(&gelu_new));
|
||||
ops.impl("gelu_fast", TORCH_BOX(&gelu_fast));
|
||||
ops.impl("gelu_quick", TORCH_BOX(&gelu_quick));
|
||||
@@ -823,6 +902,15 @@ STABLE_TORCH_LIBRARY_FRAGMENT(_C_cache_ops, ops) {
|
||||
" str kv_cache_dtype,"
|
||||
" Tensor scale) -> ()");
|
||||
|
||||
// Grouped concat_and_cache_mla across all layers (bf16 only). Each
|
||||
// layer's cache base pointer is read from kv_cache_ptrs.
|
||||
ops.def(
|
||||
"concat_and_cache_mla_grouped(Tensor kv_c, Tensor k_pe,"
|
||||
" Tensor kv_cache_ptrs,"
|
||||
" Tensor slot_mapping,"
|
||||
" int block_size, int block_stride,"
|
||||
" int entry_stride) -> ()");
|
||||
|
||||
// Rotate Q and K, then write to kv cache for MLA
|
||||
ops.def(
|
||||
"concat_and_cache_mla_rope_fused("
|
||||
@@ -921,6 +1009,8 @@ STABLE_TORCH_LIBRARY_IMPL(_C_cache_ops, CUDA, ops) {
|
||||
ops.impl("reshape_and_cache", TORCH_BOX(&reshape_and_cache));
|
||||
ops.impl("reshape_and_cache_flash", TORCH_BOX(&reshape_and_cache_flash));
|
||||
ops.impl("concat_and_cache_mla", TORCH_BOX(&concat_and_cache_mla));
|
||||
ops.impl("concat_and_cache_mla_grouped",
|
||||
TORCH_BOX(&concat_and_cache_mla_grouped));
|
||||
ops.impl("concat_and_cache_mla_rope_fused",
|
||||
TORCH_BOX(&concat_and_cache_mla_rope_fused));
|
||||
ops.impl("convert_fp8", TORCH_BOX(&convert_fp8));
|
||||
|
||||
Reference in New Issue
Block a user