[New model] Kimi K3 (#50000)

Signed-off-by: zjy0516 <riverclouds.zhu@qq.com>
Signed-off-by: Isotr0py <Isotr0py@outlook.com>
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
Signed-off-by: Jee Jee Li <jeejeelee@inferact.ai>
Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai>
Signed-off-by: Yongye Zhu <zyy1102000@gmail.com>
Signed-off-by: wzhao18 <wzhao18.sz@gmail.com>
Co-authored-by: Isotr0py <Isotr0py@outlook.com>
Co-authored-by: Thien Tran <gau.nernst@yahoo.com.sg>
Co-authored-by: Bugen Zhao <i@bugenzhao.com>
Co-authored-by: Yongye Zhu <zyy1102000@gmail.com>
Co-authored-by: Ziming Huang <zelda.huanghuang@gmail.com>
Co-authored-by: Roger Wang <hey@rogerw.io>
Co-authored-by: Isotr0py <mozf@inferact.ai>
Co-authored-by: aoshen02 <aoshen02@users.noreply.github.com>
Co-authored-by: Woosuk Kwon <woosuk@inferact.ai>
Co-authored-by: Jee Jee Li <jeejeelee@inferact.ai>
Co-authored-by: aoshen02 <aoshen@inferact.ai>
Co-authored-by: Summer Yang <girasoleyang@gmail.com>
Co-authored-by: Kevin H. Luu <khluu000@gmail.com>
Co-authored-by: Bowen Wang <abmfy@icloud.com>
Co-authored-by: gnovack <novackgm@gmail.com>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai>
Co-authored-by: xiaozhoupy <peiyuanzhou1994@gmail.com>
Co-authored-by: Roy Wang <yasong.wang@inferact.ai>
Co-authored-by: Jeff (Junze) Ma <93145857+majunze2001@users.noreply.github.com>
Co-authored-by: Codex <codex@openai.com>
Co-authored-by: Wei Zhao <51183510+wzhao18@users.noreply.github.com>
This commit is contained in:
Jiangyun Zhu
2026-07-30 03:49:58 -07:00
committed by GitHub
co-authored by Isotr0py Thien Tran Bugen Zhao Yongye Zhu Ziming Huang Roger Wang Isotr0py aoshen02 Woosuk Kwon Jee Jee Li aoshen02 Summer Yang Kevin H. Luu Bowen Wang gnovack Nick Hill Yifan Qiao xiaozhoupy Roy Wang Jeff Ma Codex Wei Zhao
parent 0c64be8873
commit aeeb36b1f1
82 changed files with 2927 additions and 1345 deletions
@@ -3,6 +3,7 @@
import torch
import vllm.envs as envs
from vllm.logger import init_logger
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.fused_moe import (
@@ -17,6 +18,7 @@ from vllm.model_executor.layers.fused_moe import modular_kernel as mk
from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import (
TRITON_BACKENDS,
Mxfp4MoeBackend,
backend_to_kernel_cls,
convert_gpt_oss_weight_to_mxfp4_moe_kernel_format,
convert_weight_to_mxfp4_moe_kernel_format,
make_mxfp4_moe_kernel,
@@ -68,8 +70,11 @@ class Mxfp4Config(QuantizationConfig):
def get_config_filenames(cls) -> list[str]:
return []
# TODO (zyongye) This is only temporaty fallback.
# We should have `Mxfp4MoEMethod` after this migration is complete.
def _make_moe_method(self, moe: FusedMoEConfig) -> FusedMoEMethodBase:
"""MoE method for RoutedExperts. Subclasses override to pick a
checkpoint-specific kernel family."""
return Mxfp4MoEMethod(moe)
def get_quant_method(
self, layer: torch.nn.Module, prefix: str
) -> "QuantizeMethodBase | None":
@@ -86,7 +91,7 @@ class Mxfp4Config(QuantizationConfig):
)
return UnquantizedLinearMethod()
elif isinstance(layer, RoutedExperts):
return GptOssMxfp4MoEMethod(layer.moe_config)
return self._make_moe_method(layer.moe_config)
elif isinstance(layer, Attention):
logger.debug_once(
"MXFP4 attention layer is not implemented. "
@@ -131,6 +136,9 @@ class GptOssMxfp4Config(Mxfp4Config):
return None
return "gpt_oss_mxfp4"
def _make_moe_method(self, moe: FusedMoEConfig) -> FusedMoEMethodBase:
return GptOssMxfp4MoEMethod(moe)
class GptOssMxfp4MoEMethod(FusedMoEMethodBase):
"""MXFP4 MoE quantization method."""
@@ -475,13 +483,45 @@ class GptOssMxfp4MoEMethod(FusedMoEMethodBase):
)
def _use_k3_situ_aiter(moe: FusedMoEConfig) -> bool:
"""Whether Kimi-K3's SiTU MXFP4 MoE should use the AITER A16W4 kernel.
K3 is weight-only MXFP4 (W4A16) with SiTU activation, which the generic
MXFP4 backend selector does not cover; route it to AITER on gfx950.
"""
from vllm.platforms import current_platform
if not current_platform.is_rocm():
return False
from vllm._aiter_ops import rocm_aiter_ops
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
from vllm.platforms.rocm import on_gfx950
return (
rocm_aiter_ops.is_fused_moe_enabled()
and on_gfx950()
and moe.activation == MoEActivation.SITU
and moe.activation_situ_linear_beta is not None
and rocm_aiter_ops.get_aiter_activation_type("situ") is not None
)
class Mxfp4MoEMethod(FusedMoEMethodBase):
"""MXFP4 MoE quantization method."""
def __init__(self, moe: FusedMoEConfig):
super().__init__(moe)
self.weight_dtype = "mxfp4"
self.mxfp4_backend, self.experts_cls = select_deepseek_v4_mxfp4_moe_backend(moe)
self.is_k3_situ_aiter = _use_k3_situ_aiter(moe)
self.experts_cls: type[mk.FusedMoEExperts] | None
if self.is_k3_situ_aiter:
self.mxfp4_backend = Mxfp4MoeBackend.AITER_MXFP4_BF16
self.experts_cls = backend_to_kernel_cls(self.mxfp4_backend)[0]
logger.info_once("Using AITER_MXFP4_BF16 for Kimi-K3 SiTU MXFP4 MoE.")
else:
self.mxfp4_backend, self.experts_cls = select_deepseek_v4_mxfp4_moe_backend(
moe
)
self.max_capture_size = moe.max_capture_size
@@ -523,6 +563,11 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
act_dtype=act_dtype,
moe_parallel_config=moe_parallel_config,
)
if self.is_k3_situ_aiter:
# K3's AITER A16W4 kernel handles K3's native intermediate size
# (moe_intermediate 3072; e.g. 384/partition at TP8); the generic
# 256 round-up would inflate weights and OOM.
return hidden_size, intermediate_size_per_partition
return mxfp4_round_up_hidden_size_and_intermediate_size(
self.mxfp4_backend, hidden_size, intermediate_size_per_partition
)
@@ -724,7 +769,58 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
layer=layer,
)
def _setup_kernel_k3_situ(self, layer: RoutedExperts) -> None:
# K3's AITER A16W4 kernel wants the separated ([gate_all, up_all])
# stage-1 layout, unlike the interleaved gpt-oss/DeepSeek path in
# convert_weight_to_mxfp4_moe_kernel_format. Preshuffle once here.
from aiter.utility.fp4_utils import e8m0_shuffle
from vllm._aiter_ops import rocm_aiter_ops
fp4_dtype = torch.float4_e2m1fn_x2
e8m0_dtype = torch.float8_e8m0fnu
num_experts = layer.w13_weight.shape[0]
# a8w4 (AITER_SITUV2_A8W4=1) uses the gate/up-interleaved (_gui_) fp8
# flydsl kernels, which need w13 weight+scale in interleave layout.
# Default a16w4 keeps the separated layout.
guinterleave = envs.AITER_SITUV2_A8W4
w13 = rocm_aiter_ops.shuffle_weight_a16w4(
layer.w13_weight.data.view(fp4_dtype), 16, guinterleave
)
w2 = rocm_aiter_ops.shuffle_weight_a16w4(
layer.w2_weight.data.view(fp4_dtype), 16, False
)
w13_scale_raw = layer.w13_weight_scale.data.view(e8m0_dtype)
w2_scale_raw = layer.w2_weight_scale.data.view(e8m0_dtype)
w13_scale = rocm_aiter_ops.shuffle_scale_a16w4(
w13_scale_raw.view(-1, w13_scale_raw.shape[-1]), num_experts, guinterleave
)
w2_scale = e8m0_shuffle(w2_scale_raw.view(-1, w2_scale_raw.shape[-1]))
replace_parameter(layer, "w13_weight", w13)
replace_parameter(layer, "w2_weight", w2)
replace_parameter(layer, "w13_weight_scale", w13_scale)
replace_parameter(layer, "w2_weight_scale", w2_scale)
layer.w13_weight.is_shuffled = True
layer.w2_weight.is_shuffled = True
self.moe_quant_config = self.get_fused_moe_quant_config(layer)
if self.moe_quant_config is not None and self.experts_cls is not None:
self.moe_kernel = make_mxfp4_moe_kernel(
moe_quant_config=self.moe_quant_config,
moe_config=self.moe,
mxfp4_backend=self.mxfp4_backend,
experts_cls=self.experts_cls,
routing_tables=layer._expert_routing_tables(),
layer=layer,
)
def process_weights_after_loading(self, layer):
if self.is_k3_situ_aiter:
self._setup_kernel_k3_situ(layer)
return
w13 = layer.w13_weight
w2 = layer.w2_weight
w13_scale = layer.w13_weight_scale
@@ -819,4 +915,8 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
global_num_experts=layer.global_num_experts,
expert_map=layer.expert_map,
apply_router_weight_on_input=layer.apply_router_weight_on_input,
num_expert_group=layer.num_expert_group,
topk_group=layer.topk_group,
e_score_correction_bias=layer.e_score_correction_bias,
routed_scaling_factor=layer.routed_scaling_factor,
)
@@ -18,20 +18,34 @@ def activation_to_flashinfer_int(activation: MoEActivation) -> int:
return activation_to_flashinfer_type(activation).value
def has_flashinfer_situ_activation() -> bool:
try:
from flashinfer.fused_moe.core import ActivationType
except ImportError:
return False
return hasattr(ActivationType, "Situ")
def activation_to_flashinfer_type(activation: MoEActivation) -> "ActivationType":
from flashinfer.fused_moe.core import ActivationType
if activation == MoEActivation.SITU:
situ = getattr(ActivationType, "Situ", None)
if situ is None:
raise ValueError("The installed FlashInfer does not support SITU")
return situ
# silu and gelu are mapped to their gated versions SwiGLU and GeGLU respectively
ACTIVATION_TO_FI_ACTIVATION = {
MoEActivation.SILU_NO_MUL: ActivationType.Silu,
MoEActivation.GELU_NO_MUL: ActivationType.Gelu,
MoEActivation.SILU: ActivationType.Swiglu,
# SwiGLU-OAI uses Swiglu; the OAI alpha/beta/clamp come from gemm1_* args.
MoEActivation.SWIGLUOAI: ActivationType.Swiglu,
MoEActivation.SWIGLUOAI_UNINTERLEAVE: ActivationType.Swiglu,
MoEActivation.GELU: ActivationType.Geglu,
MoEActivation.GELU_TANH: ActivationType.Geglu,
MoEActivation.RELU2_NO_MUL: ActivationType.Relu2,
MoEActivation.SWIGLUOAI_UNINTERLEAVE: ActivationType.Swiglu,
}
return ACTIVATION_TO_FI_ACTIVATION[activation]
@@ -474,7 +474,11 @@ def prepare_moe_fp4_layer_for_marlin(
group_size = 16 if is_nvfp4 else 32
e = layer.moe_config.num_experts
# Use the per-rank (local) expert count: under expert parallelism the
# w13_weight/w2_weight tensors only hold this rank's experts (created with
# local_num_experts), whereas moe_config.num_experts is the global count.
# With no EP the two are equal, so the non-EP path is unchanged.
e = layer.moe_config.num_local_experts
k = layer.moe_config.hidden_dim
n = layer.moe_config.intermediate_size_per_partition
@@ -41,8 +41,8 @@ def _mxfp8_e4m3_quantize_torch(
) -> tuple[torch.Tensor, torch.Tensor]:
"""Naive MXFP8 quantization.
For each block of 32 elements along the last dimension, compute a
shared e8m0 scale (the biased exponent of the block-wise amax)
and quantize each element to float8_e4m3fn.
shared e8m0 scale that fits the block-wise amax into the finite
float8_e4m3fn range, and quantize each element to float8_e4m3fn.
Returns (quantized_values [same shape, fp8], scales uint8).
Scale shape depends on is_sf_swizzled_layout:
@@ -58,7 +58,8 @@ def _mxfp8_e4m3_quantize_torch(
amax = x_blocked.abs().amax(dim=-1)
amax = amax.clamp(min=torch.finfo(torch.float32).tiny)
scale_biased = torch.floor(torch.log2(amax)) + 127.0
fp8_max = torch.finfo(MXFP8_VALUE_DTYPE).max
scale_biased = torch.ceil(torch.log2(amax / fp8_max)) + 127.0
scale_biased = scale_biased.clamp(0, 254)
scales_uint8 = scale_biased.to(torch.uint8)