[New model] Kimi K3 (#50000)
Signed-off-by: zjy0516 <riverclouds.zhu@qq.com> Signed-off-by: Isotr0py <Isotr0py@outlook.com> Signed-off-by: Bugen Zhao <i@bugenzhao.com> Signed-off-by: Jee Jee Li <jeejeelee@inferact.ai> Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai> Signed-off-by: Yongye Zhu <zyy1102000@gmail.com> Signed-off-by: wzhao18 <wzhao18.sz@gmail.com> Co-authored-by: Isotr0py <Isotr0py@outlook.com> Co-authored-by: Thien Tran <gau.nernst@yahoo.com.sg> Co-authored-by: Bugen Zhao <i@bugenzhao.com> Co-authored-by: Yongye Zhu <zyy1102000@gmail.com> Co-authored-by: Ziming Huang <zelda.huanghuang@gmail.com> Co-authored-by: Roger Wang <hey@rogerw.io> Co-authored-by: Isotr0py <mozf@inferact.ai> Co-authored-by: aoshen02 <aoshen02@users.noreply.github.com> Co-authored-by: Woosuk Kwon <woosuk@inferact.ai> Co-authored-by: Jee Jee Li <jeejeelee@inferact.ai> Co-authored-by: aoshen02 <aoshen@inferact.ai> Co-authored-by: Summer Yang <girasoleyang@gmail.com> Co-authored-by: Kevin H. Luu <khluu000@gmail.com> Co-authored-by: Bowen Wang <abmfy@icloud.com> Co-authored-by: gnovack <novackgm@gmail.com> Co-authored-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai> Co-authored-by: xiaozhoupy <peiyuanzhou1994@gmail.com> Co-authored-by: Roy Wang <yasong.wang@inferact.ai> Co-authored-by: Jeff (Junze) Ma <93145857+majunze2001@users.noreply.github.com> Co-authored-by: Codex <codex@openai.com> Co-authored-by: Wei Zhao <51183510+wzhao18@users.noreply.github.com>
This commit is contained in:
co-authored by
Isotr0py
Thien Tran
Bugen Zhao
Yongye Zhu
Ziming Huang
Roger Wang
Isotr0py
aoshen02
Woosuk Kwon
Jee Jee Li
aoshen02
Summer Yang
Kevin H. Luu
Bowen Wang
gnovack
Nick Hill
Yifan Qiao
xiaozhoupy
Roy Wang
Jeff Ma
Codex
Wei Zhao
parent
0c64be8873
commit
aeeb36b1f1
@@ -18,20 +18,34 @@ def activation_to_flashinfer_int(activation: MoEActivation) -> int:
|
||||
return activation_to_flashinfer_type(activation).value
|
||||
|
||||
|
||||
def has_flashinfer_situ_activation() -> bool:
|
||||
try:
|
||||
from flashinfer.fused_moe.core import ActivationType
|
||||
except ImportError:
|
||||
return False
|
||||
return hasattr(ActivationType, "Situ")
|
||||
|
||||
|
||||
def activation_to_flashinfer_type(activation: MoEActivation) -> "ActivationType":
|
||||
from flashinfer.fused_moe.core import ActivationType
|
||||
|
||||
if activation == MoEActivation.SITU:
|
||||
situ = getattr(ActivationType, "Situ", None)
|
||||
if situ is None:
|
||||
raise ValueError("The installed FlashInfer does not support SITU")
|
||||
return situ
|
||||
|
||||
# silu and gelu are mapped to their gated versions SwiGLU and GeGLU respectively
|
||||
ACTIVATION_TO_FI_ACTIVATION = {
|
||||
MoEActivation.SILU_NO_MUL: ActivationType.Silu,
|
||||
MoEActivation.GELU_NO_MUL: ActivationType.Gelu,
|
||||
MoEActivation.SILU: ActivationType.Swiglu,
|
||||
# SwiGLU-OAI uses Swiglu; the OAI alpha/beta/clamp come from gemm1_* args.
|
||||
MoEActivation.SWIGLUOAI: ActivationType.Swiglu,
|
||||
MoEActivation.SWIGLUOAI_UNINTERLEAVE: ActivationType.Swiglu,
|
||||
MoEActivation.GELU: ActivationType.Geglu,
|
||||
MoEActivation.GELU_TANH: ActivationType.Geglu,
|
||||
MoEActivation.RELU2_NO_MUL: ActivationType.Relu2,
|
||||
MoEActivation.SWIGLUOAI_UNINTERLEAVE: ActivationType.Swiglu,
|
||||
}
|
||||
return ACTIVATION_TO_FI_ACTIVATION[activation]
|
||||
|
||||
|
||||
@@ -474,7 +474,11 @@ def prepare_moe_fp4_layer_for_marlin(
|
||||
|
||||
group_size = 16 if is_nvfp4 else 32
|
||||
|
||||
e = layer.moe_config.num_experts
|
||||
# Use the per-rank (local) expert count: under expert parallelism the
|
||||
# w13_weight/w2_weight tensors only hold this rank's experts (created with
|
||||
# local_num_experts), whereas moe_config.num_experts is the global count.
|
||||
# With no EP the two are equal, so the non-EP path is unchanged.
|
||||
e = layer.moe_config.num_local_experts
|
||||
k = layer.moe_config.hidden_dim
|
||||
n = layer.moe_config.intermediate_size_per_partition
|
||||
|
||||
|
||||
@@ -41,8 +41,8 @@ def _mxfp8_e4m3_quantize_torch(
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Naive MXFP8 quantization.
|
||||
For each block of 32 elements along the last dimension, compute a
|
||||
shared e8m0 scale (the biased exponent of the block-wise amax)
|
||||
and quantize each element to float8_e4m3fn.
|
||||
shared e8m0 scale that fits the block-wise amax into the finite
|
||||
float8_e4m3fn range, and quantize each element to float8_e4m3fn.
|
||||
|
||||
Returns (quantized_values [same shape, fp8], scales uint8).
|
||||
Scale shape depends on is_sf_swizzled_layout:
|
||||
@@ -58,7 +58,8 @@ def _mxfp8_e4m3_quantize_torch(
|
||||
|
||||
amax = x_blocked.abs().amax(dim=-1)
|
||||
amax = amax.clamp(min=torch.finfo(torch.float32).tiny)
|
||||
scale_biased = torch.floor(torch.log2(amax)) + 127.0
|
||||
fp8_max = torch.finfo(MXFP8_VALUE_DTYPE).max
|
||||
scale_biased = torch.ceil(torch.log2(amax / fp8_max)) + 127.0
|
||||
scale_biased = scale_biased.clamp(0, 254)
|
||||
scales_uint8 = scale_biased.to(torch.uint8)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user