[New model] Kimi K3 (#50000)
Signed-off-by: zjy0516 <riverclouds.zhu@qq.com> Signed-off-by: Isotr0py <Isotr0py@outlook.com> Signed-off-by: Bugen Zhao <i@bugenzhao.com> Signed-off-by: Jee Jee Li <jeejeelee@inferact.ai> Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai> Signed-off-by: Yongye Zhu <zyy1102000@gmail.com> Signed-off-by: wzhao18 <wzhao18.sz@gmail.com> Co-authored-by: Isotr0py <Isotr0py@outlook.com> Co-authored-by: Thien Tran <gau.nernst@yahoo.com.sg> Co-authored-by: Bugen Zhao <i@bugenzhao.com> Co-authored-by: Yongye Zhu <zyy1102000@gmail.com> Co-authored-by: Ziming Huang <zelda.huanghuang@gmail.com> Co-authored-by: Roger Wang <hey@rogerw.io> Co-authored-by: Isotr0py <mozf@inferact.ai> Co-authored-by: aoshen02 <aoshen02@users.noreply.github.com> Co-authored-by: Woosuk Kwon <woosuk@inferact.ai> Co-authored-by: Jee Jee Li <jeejeelee@inferact.ai> Co-authored-by: aoshen02 <aoshen@inferact.ai> Co-authored-by: Summer Yang <girasoleyang@gmail.com> Co-authored-by: Kevin H. Luu <khluu000@gmail.com> Co-authored-by: Bowen Wang <abmfy@icloud.com> Co-authored-by: gnovack <novackgm@gmail.com> Co-authored-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai> Co-authored-by: xiaozhoupy <peiyuanzhou1994@gmail.com> Co-authored-by: Roy Wang <yasong.wang@inferact.ai> Co-authored-by: Jeff (Junze) Ma <93145857+majunze2001@users.noreply.github.com> Co-authored-by: Codex <codex@openai.com> Co-authored-by: Wei Zhao <51183510+wzhao18@users.noreply.github.com>
This commit is contained in:
co-authored by
Isotr0py
Thien Tran
Bugen Zhao
Yongye Zhu
Ziming Huang
Roger Wang
Isotr0py
aoshen02
Woosuk Kwon
Jee Jee Li
aoshen02
Summer Yang
Kevin H. Luu
Bowen Wang
gnovack
Nick Hill
Yifan Qiao
xiaozhoupy
Roy Wang
Jeff Ma
Codex
Wei Zhao
parent
0c64be8873
commit
aeeb36b1f1
@@ -3,6 +3,7 @@
|
||||
|
||||
import torch
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.attention import Attention
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
@@ -17,6 +18,7 @@ from vllm.model_executor.layers.fused_moe import modular_kernel as mk
|
||||
from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import (
|
||||
TRITON_BACKENDS,
|
||||
Mxfp4MoeBackend,
|
||||
backend_to_kernel_cls,
|
||||
convert_gpt_oss_weight_to_mxfp4_moe_kernel_format,
|
||||
convert_weight_to_mxfp4_moe_kernel_format,
|
||||
make_mxfp4_moe_kernel,
|
||||
@@ -68,8 +70,11 @@ class Mxfp4Config(QuantizationConfig):
|
||||
def get_config_filenames(cls) -> list[str]:
|
||||
return []
|
||||
|
||||
# TODO (zyongye) This is only temporaty fallback.
|
||||
# We should have `Mxfp4MoEMethod` after this migration is complete.
|
||||
def _make_moe_method(self, moe: FusedMoEConfig) -> FusedMoEMethodBase:
|
||||
"""MoE method for RoutedExperts. Subclasses override to pick a
|
||||
checkpoint-specific kernel family."""
|
||||
return Mxfp4MoEMethod(moe)
|
||||
|
||||
def get_quant_method(
|
||||
self, layer: torch.nn.Module, prefix: str
|
||||
) -> "QuantizeMethodBase | None":
|
||||
@@ -86,7 +91,7 @@ class Mxfp4Config(QuantizationConfig):
|
||||
)
|
||||
return UnquantizedLinearMethod()
|
||||
elif isinstance(layer, RoutedExperts):
|
||||
return GptOssMxfp4MoEMethod(layer.moe_config)
|
||||
return self._make_moe_method(layer.moe_config)
|
||||
elif isinstance(layer, Attention):
|
||||
logger.debug_once(
|
||||
"MXFP4 attention layer is not implemented. "
|
||||
@@ -131,6 +136,9 @@ class GptOssMxfp4Config(Mxfp4Config):
|
||||
return None
|
||||
return "gpt_oss_mxfp4"
|
||||
|
||||
def _make_moe_method(self, moe: FusedMoEConfig) -> FusedMoEMethodBase:
|
||||
return GptOssMxfp4MoEMethod(moe)
|
||||
|
||||
|
||||
class GptOssMxfp4MoEMethod(FusedMoEMethodBase):
|
||||
"""MXFP4 MoE quantization method."""
|
||||
@@ -475,13 +483,45 @@ class GptOssMxfp4MoEMethod(FusedMoEMethodBase):
|
||||
)
|
||||
|
||||
|
||||
def _use_k3_situ_aiter(moe: FusedMoEConfig) -> bool:
|
||||
"""Whether Kimi-K3's SiTU MXFP4 MoE should use the AITER A16W4 kernel.
|
||||
|
||||
K3 is weight-only MXFP4 (W4A16) with SiTU activation, which the generic
|
||||
MXFP4 backend selector does not cover; route it to AITER on gfx950.
|
||||
"""
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
if not current_platform.is_rocm():
|
||||
return False
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
from vllm.platforms.rocm import on_gfx950
|
||||
|
||||
return (
|
||||
rocm_aiter_ops.is_fused_moe_enabled()
|
||||
and on_gfx950()
|
||||
and moe.activation == MoEActivation.SITU
|
||||
and moe.activation_situ_linear_beta is not None
|
||||
and rocm_aiter_ops.get_aiter_activation_type("situ") is not None
|
||||
)
|
||||
|
||||
|
||||
class Mxfp4MoEMethod(FusedMoEMethodBase):
|
||||
"""MXFP4 MoE quantization method."""
|
||||
|
||||
def __init__(self, moe: FusedMoEConfig):
|
||||
super().__init__(moe)
|
||||
self.weight_dtype = "mxfp4"
|
||||
self.mxfp4_backend, self.experts_cls = select_deepseek_v4_mxfp4_moe_backend(moe)
|
||||
self.is_k3_situ_aiter = _use_k3_situ_aiter(moe)
|
||||
self.experts_cls: type[mk.FusedMoEExperts] | None
|
||||
if self.is_k3_situ_aiter:
|
||||
self.mxfp4_backend = Mxfp4MoeBackend.AITER_MXFP4_BF16
|
||||
self.experts_cls = backend_to_kernel_cls(self.mxfp4_backend)[0]
|
||||
logger.info_once("Using AITER_MXFP4_BF16 for Kimi-K3 SiTU MXFP4 MoE.")
|
||||
else:
|
||||
self.mxfp4_backend, self.experts_cls = select_deepseek_v4_mxfp4_moe_backend(
|
||||
moe
|
||||
)
|
||||
|
||||
self.max_capture_size = moe.max_capture_size
|
||||
|
||||
@@ -523,6 +563,11 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
|
||||
act_dtype=act_dtype,
|
||||
moe_parallel_config=moe_parallel_config,
|
||||
)
|
||||
if self.is_k3_situ_aiter:
|
||||
# K3's AITER A16W4 kernel handles K3's native intermediate size
|
||||
# (moe_intermediate 3072; e.g. 384/partition at TP8); the generic
|
||||
# 256 round-up would inflate weights and OOM.
|
||||
return hidden_size, intermediate_size_per_partition
|
||||
return mxfp4_round_up_hidden_size_and_intermediate_size(
|
||||
self.mxfp4_backend, hidden_size, intermediate_size_per_partition
|
||||
)
|
||||
@@ -724,7 +769,58 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
|
||||
layer=layer,
|
||||
)
|
||||
|
||||
def _setup_kernel_k3_situ(self, layer: RoutedExperts) -> None:
|
||||
# K3's AITER A16W4 kernel wants the separated ([gate_all, up_all])
|
||||
# stage-1 layout, unlike the interleaved gpt-oss/DeepSeek path in
|
||||
# convert_weight_to_mxfp4_moe_kernel_format. Preshuffle once here.
|
||||
from aiter.utility.fp4_utils import e8m0_shuffle
|
||||
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
|
||||
fp4_dtype = torch.float4_e2m1fn_x2
|
||||
e8m0_dtype = torch.float8_e8m0fnu
|
||||
num_experts = layer.w13_weight.shape[0]
|
||||
|
||||
# a8w4 (AITER_SITUV2_A8W4=1) uses the gate/up-interleaved (_gui_) fp8
|
||||
# flydsl kernels, which need w13 weight+scale in interleave layout.
|
||||
# Default a16w4 keeps the separated layout.
|
||||
guinterleave = envs.AITER_SITUV2_A8W4
|
||||
w13 = rocm_aiter_ops.shuffle_weight_a16w4(
|
||||
layer.w13_weight.data.view(fp4_dtype), 16, guinterleave
|
||||
)
|
||||
w2 = rocm_aiter_ops.shuffle_weight_a16w4(
|
||||
layer.w2_weight.data.view(fp4_dtype), 16, False
|
||||
)
|
||||
w13_scale_raw = layer.w13_weight_scale.data.view(e8m0_dtype)
|
||||
w2_scale_raw = layer.w2_weight_scale.data.view(e8m0_dtype)
|
||||
w13_scale = rocm_aiter_ops.shuffle_scale_a16w4(
|
||||
w13_scale_raw.view(-1, w13_scale_raw.shape[-1]), num_experts, guinterleave
|
||||
)
|
||||
w2_scale = e8m0_shuffle(w2_scale_raw.view(-1, w2_scale_raw.shape[-1]))
|
||||
|
||||
replace_parameter(layer, "w13_weight", w13)
|
||||
replace_parameter(layer, "w2_weight", w2)
|
||||
replace_parameter(layer, "w13_weight_scale", w13_scale)
|
||||
replace_parameter(layer, "w2_weight_scale", w2_scale)
|
||||
layer.w13_weight.is_shuffled = True
|
||||
layer.w2_weight.is_shuffled = True
|
||||
|
||||
self.moe_quant_config = self.get_fused_moe_quant_config(layer)
|
||||
if self.moe_quant_config is not None and self.experts_cls is not None:
|
||||
self.moe_kernel = make_mxfp4_moe_kernel(
|
||||
moe_quant_config=self.moe_quant_config,
|
||||
moe_config=self.moe,
|
||||
mxfp4_backend=self.mxfp4_backend,
|
||||
experts_cls=self.experts_cls,
|
||||
routing_tables=layer._expert_routing_tables(),
|
||||
layer=layer,
|
||||
)
|
||||
|
||||
def process_weights_after_loading(self, layer):
|
||||
if self.is_k3_situ_aiter:
|
||||
self._setup_kernel_k3_situ(layer)
|
||||
return
|
||||
|
||||
w13 = layer.w13_weight
|
||||
w2 = layer.w2_weight
|
||||
w13_scale = layer.w13_weight_scale
|
||||
@@ -819,4 +915,8 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
|
||||
global_num_experts=layer.global_num_experts,
|
||||
expert_map=layer.expert_map,
|
||||
apply_router_weight_on_input=layer.apply_router_weight_on_input,
|
||||
num_expert_group=layer.num_expert_group,
|
||||
topk_group=layer.topk_group,
|
||||
e_score_correction_bias=layer.e_score_correction_bias,
|
||||
routed_scaling_factor=layer.routed_scaling_factor,
|
||||
)
|
||||
|
||||
@@ -18,20 +18,34 @@ def activation_to_flashinfer_int(activation: MoEActivation) -> int:
|
||||
return activation_to_flashinfer_type(activation).value
|
||||
|
||||
|
||||
def has_flashinfer_situ_activation() -> bool:
|
||||
try:
|
||||
from flashinfer.fused_moe.core import ActivationType
|
||||
except ImportError:
|
||||
return False
|
||||
return hasattr(ActivationType, "Situ")
|
||||
|
||||
|
||||
def activation_to_flashinfer_type(activation: MoEActivation) -> "ActivationType":
|
||||
from flashinfer.fused_moe.core import ActivationType
|
||||
|
||||
if activation == MoEActivation.SITU:
|
||||
situ = getattr(ActivationType, "Situ", None)
|
||||
if situ is None:
|
||||
raise ValueError("The installed FlashInfer does not support SITU")
|
||||
return situ
|
||||
|
||||
# silu and gelu are mapped to their gated versions SwiGLU and GeGLU respectively
|
||||
ACTIVATION_TO_FI_ACTIVATION = {
|
||||
MoEActivation.SILU_NO_MUL: ActivationType.Silu,
|
||||
MoEActivation.GELU_NO_MUL: ActivationType.Gelu,
|
||||
MoEActivation.SILU: ActivationType.Swiglu,
|
||||
# SwiGLU-OAI uses Swiglu; the OAI alpha/beta/clamp come from gemm1_* args.
|
||||
MoEActivation.SWIGLUOAI: ActivationType.Swiglu,
|
||||
MoEActivation.SWIGLUOAI_UNINTERLEAVE: ActivationType.Swiglu,
|
||||
MoEActivation.GELU: ActivationType.Geglu,
|
||||
MoEActivation.GELU_TANH: ActivationType.Geglu,
|
||||
MoEActivation.RELU2_NO_MUL: ActivationType.Relu2,
|
||||
MoEActivation.SWIGLUOAI_UNINTERLEAVE: ActivationType.Swiglu,
|
||||
}
|
||||
return ACTIVATION_TO_FI_ACTIVATION[activation]
|
||||
|
||||
|
||||
@@ -474,7 +474,11 @@ def prepare_moe_fp4_layer_for_marlin(
|
||||
|
||||
group_size = 16 if is_nvfp4 else 32
|
||||
|
||||
e = layer.moe_config.num_experts
|
||||
# Use the per-rank (local) expert count: under expert parallelism the
|
||||
# w13_weight/w2_weight tensors only hold this rank's experts (created with
|
||||
# local_num_experts), whereas moe_config.num_experts is the global count.
|
||||
# With no EP the two are equal, so the non-EP path is unchanged.
|
||||
e = layer.moe_config.num_local_experts
|
||||
k = layer.moe_config.hidden_dim
|
||||
n = layer.moe_config.intermediate_size_per_partition
|
||||
|
||||
|
||||
@@ -41,8 +41,8 @@ def _mxfp8_e4m3_quantize_torch(
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Naive MXFP8 quantization.
|
||||
For each block of 32 elements along the last dimension, compute a
|
||||
shared e8m0 scale (the biased exponent of the block-wise amax)
|
||||
and quantize each element to float8_e4m3fn.
|
||||
shared e8m0 scale that fits the block-wise amax into the finite
|
||||
float8_e4m3fn range, and quantize each element to float8_e4m3fn.
|
||||
|
||||
Returns (quantized_values [same shape, fp8], scales uint8).
|
||||
Scale shape depends on is_sf_swizzled_layout:
|
||||
@@ -58,7 +58,8 @@ def _mxfp8_e4m3_quantize_torch(
|
||||
|
||||
amax = x_blocked.abs().amax(dim=-1)
|
||||
amax = amax.clamp(min=torch.finfo(torch.float32).tiny)
|
||||
scale_biased = torch.floor(torch.log2(amax)) + 127.0
|
||||
fp8_max = torch.finfo(MXFP8_VALUE_DTYPE).max
|
||||
scale_biased = torch.ceil(torch.log2(amax / fp8_max)) + 127.0
|
||||
scale_biased = scale_biased.clamp(0, 254)
|
||||
scales_uint8 = scale_biased.to(torch.uint8)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user