[New model] Kimi K3 (#50000)

Signed-off-by: zjy0516 <riverclouds.zhu@qq.com>
Signed-off-by: Isotr0py <Isotr0py@outlook.com>
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
Signed-off-by: Jee Jee Li <jeejeelee@inferact.ai>
Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai>
Signed-off-by: Yongye Zhu <zyy1102000@gmail.com>
Signed-off-by: wzhao18 <wzhao18.sz@gmail.com>
Co-authored-by: Isotr0py <Isotr0py@outlook.com>
Co-authored-by: Thien Tran <gau.nernst@yahoo.com.sg>
Co-authored-by: Bugen Zhao <i@bugenzhao.com>
Co-authored-by: Yongye Zhu <zyy1102000@gmail.com>
Co-authored-by: Ziming Huang <zelda.huanghuang@gmail.com>
Co-authored-by: Roger Wang <hey@rogerw.io>
Co-authored-by: Isotr0py <mozf@inferact.ai>
Co-authored-by: aoshen02 <aoshen02@users.noreply.github.com>
Co-authored-by: Woosuk Kwon <woosuk@inferact.ai>
Co-authored-by: Jee Jee Li <jeejeelee@inferact.ai>
Co-authored-by: aoshen02 <aoshen@inferact.ai>
Co-authored-by: Summer Yang <girasoleyang@gmail.com>
Co-authored-by: Kevin H. Luu <khluu000@gmail.com>
Co-authored-by: Bowen Wang <abmfy@icloud.com>
Co-authored-by: gnovack <novackgm@gmail.com>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai>
Co-authored-by: xiaozhoupy <peiyuanzhou1994@gmail.com>
Co-authored-by: Roy Wang <yasong.wang@inferact.ai>
Co-authored-by: Jeff (Junze) Ma <93145857+majunze2001@users.noreply.github.com>
Co-authored-by: Codex <codex@openai.com>
Co-authored-by: Wei Zhao <51183510+wzhao18@users.noreply.github.com>
This commit is contained in:
Jiangyun Zhu
2026-07-30 03:49:58 -07:00
committed by GitHub
co-authored by Isotr0py Thien Tran Bugen Zhao Yongye Zhu Ziming Huang Roger Wang Isotr0py aoshen02 Woosuk Kwon Jee Jee Li aoshen02 Summer Yang Kevin H. Luu Bowen Wang gnovack Nick Hill Yifan Qiao xiaozhoupy Roy Wang Jeff Ma Codex Wei Zhao
parent 0c64be8873
commit aeeb36b1f1
82 changed files with 2927 additions and 1345 deletions
+35
View File
@@ -339,6 +339,39 @@ class DeepseekV4ForCausalLMConfig(VerifyAndUpdateConfig):
)
class KimiK3ForConditionalGenerationConfig(VerifyAndUpdateConfig):
"""Route MXFP4-checkpointed Kimi-K3 MoE experts to the MXFP4 interface.
Kimi-K3 ships its routed experts as compressed-tensors
``mxfp4-pack-quantized`` (``quant_method="compressed-tensors"``), which
lands them on ``CompressedTensorsW4A4Mxfp4MoEMethod`` and its narrow kernel
selection. Rewriting ``quant_method`` to ``"mxfp4"`` selects ``Mxfp4Config``
(hence ``Mxfp4MoEMethod``) with its full backend set, while any non-MXFP4
checkpoint is left untouched. Covers both the main model and the MTP draft.
``model_arch_config.quantization_config`` is a separate dict, snapshotted in
``ModelConfig.__init__`` before this hook runs, and it is what
``_verify_quantization`` reads when resolving the quant method. Patch it
alongside the hf configs so the rewrite lands before resolution; otherwise
the main model still resolves to compressed-tensors.
"""
@staticmethod
def verify_and_update_model_config(model_config: "ModelConfig") -> None:
for cfg in (
model_config.hf_config,
model_config.hf_text_config,
model_config.model_arch_config,
):
quant_config = getattr(cfg, "quantization_config", None)
if (
isinstance(quant_config, dict)
and quant_config.get("quant_method") == "compressed-tensors"
and quant_config.get("format") == "mxfp4-pack-quantized"
):
quant_config["quant_method"] = "mxfp4"
class GptOssForCausalLMConfig(VerifyAndUpdateConfig):
@staticmethod
def verify_and_update_model_config(model_config: "ModelConfig") -> None:
@@ -884,6 +917,8 @@ MODELS_CONFIG_MAP: dict[str, type[VerifyAndUpdateConfig]] = {
"JambaForSequenceClassification": JambaForSequenceClassificationConfig,
"JinaForRanking": JinaForRankingConfig,
"JinaVLForRanking": JinaVLForSequenceClassificationConfig,
"KimiK3ForConditionalGeneration": KimiK3ForConditionalGenerationConfig,
"KimiK3MTPModel": KimiK3ForConditionalGenerationConfig,
"LlamaBidirectionalForSequenceClassification": LlamaBidirectionalConfig,
"LlamaBidirectionalModel": LlamaBidirectionalConfig,
"LlamaNemotronVLForSequenceClassification": LlamaNemotronVLConfig,
@@ -983,6 +983,7 @@ class DeepseekV2MLAAttention(nn.Module):
topk_indices_buffer: torch.Tensor | None = None,
input_size: int | None = None,
reduce_results: bool = True,
non_causal_multi_token_decode: bool = False,
) -> None:
super().__init__()
self.hidden_size = hidden_size
@@ -1178,6 +1179,7 @@ class DeepseekV2MLAAttention(nn.Module):
# the V1 proposer. A frozen True would leave the draft reading a
# never-written topk buffer.
skip_topk=_skip_topk and not is_mtp_layer,
non_causal_multi_token_decode=non_causal_multi_token_decode,
# Do not skip scoring for MTP layers: their top-k buffer may be
# reused by later draft iterations through index sharing.
allow_short_prefill_indexer_scoring_skip=not is_mtp_layer,
+1 -1
View File
@@ -1407,7 +1407,7 @@ class EagleModelMixin:
aux_hidden_states: list[torch.Tensor],
layer_idx: int,
hidden_states: torch.Tensor,
residual: torch.Tensor,
residual: torch.Tensor | None,
) -> list[torch.Tensor]:
if layer_idx in self.aux_hidden_state_layers:
value = hidden_states + residual if residual is not None else hidden_states
+204 -25
View File
@@ -34,6 +34,7 @@ from vllm.model_executor.models.vision import (
is_vit_use_data_parallel,
run_dp_sharded_mrope_vision_model,
)
from vllm.platforms import current_platform
from vllm.transformers_utils.configs.kimi_k25 import KimiK25VisionConfig
logger = init_logger(__name__)
@@ -154,9 +155,7 @@ class Learnable2DInterpPosEmbDivided_fixed(nn.Module):
def reset_parameters(self):
nn.init.normal_(self.weight)
def forward(
self, x: torch.Tensor, grid_thws: torch.Tensor | list[list[int]]
) -> torch.Tensor:
def get_pos_embeds(self, grid_thws: torch.Tensor | list[list[int]]) -> torch.Tensor:
pos_embs = []
grid_thw_list = grid_thws if isinstance(grid_thws, list) else grid_thws.tolist()
for t, h, w in grid_thw_list:
@@ -179,8 +178,12 @@ class Learnable2DInterpPosEmbDivided_fixed(nn.Module):
pos_embs.append(pos_emb_3d.reshape(-1, pos_emb_3d.shape[-1]))
out = x + torch.cat(pos_embs)
return out
return torch.cat(pos_embs)
def forward(
self, x: torch.Tensor, grid_thws: torch.Tensor | list[list[int]]
) -> torch.Tensor:
return x + self.get_pos_embeds(grid_thws)
class MoonVision3dPatchEmbed(nn.Module):
@@ -195,6 +198,8 @@ class MoonVision3dPatchEmbed(nn.Module):
pos_emb_width: int = 14,
pos_emb_time: int = 4,
pos_emb_type: str = "divided_fixed",
patch_embed_proj_bias: bool = True,
pos_emb_interpolation_mode: str = "bicubic",
):
super().__init__()
assert isinstance(patch_size, int | Sequence), (
@@ -208,7 +213,11 @@ class MoonVision3dPatchEmbed(nn.Module):
self.patch_size = patch_size
self.proj = nn.Conv2d(
in_dim, out_dim, kernel_size=patch_size, stride=patch_size
in_dim,
out_dim,
kernel_size=patch_size,
stride=patch_size,
bias=patch_embed_proj_bias,
)
if pos_emb_type == "divided_fixed":
@@ -217,17 +226,37 @@ class MoonVision3dPatchEmbed(nn.Module):
width=pos_emb_width,
num_frames=pos_emb_time,
dim=out_dim,
interpolation_mode=pos_emb_interpolation_mode,
)
else:
raise NotImplementedError(f"Not support pos_emb_type: {pos_emb_type}")
def forward(
self, x: torch.Tensor, grid_thws: torch.Tensor | list[list[int]]
self,
x: torch.Tensor,
grid_thws: torch.Tensor | list[list[int]] | None,
*,
pos_embeds: torch.Tensor | None = None,
) -> torch.Tensor:
x = self.proj(x).view(x.size(0), -1)
# apply positional embedding
x = self.pos_emb(x, grid_thws)
return x
x = self._proj(x).view(x.size(0), -1)
if pos_embeds is not None:
return x + pos_embeds
assert grid_thws is not None
return self.pos_emb(x, grid_thws)
def _proj(self, x: torch.Tensor) -> torch.Tensor:
# MIOpen conv2d intermittently fails under load on ROCm; use aiter Triton.
if current_platform.is_rocm() and x.dtype in (torch.float16, torch.bfloat16):
from aiter.ops.triton.conv.conv2d import conv2d
return conv2d(
x,
self.proj.weight,
self.proj.bias,
stride=self.patch_size,
layout="nchw",
)
return self.proj(x)
class Rope2DPosEmbRepeated(nn.Module):
@@ -342,6 +371,14 @@ class MLP2(nn.Module):
return x
def _make_vision_norm(norm_type: str, hidden_dim: int) -> nn.Module:
if norm_type == "layernorm":
return nn.LayerNorm(hidden_dim)
if norm_type == "rmsnorm":
return nn.RMSNorm(hidden_dim)
raise NotImplementedError(f"Not support norm_type: {norm_type}")
class MoonViTEncoderLayer(nn.Module):
"""Single encoder layer for MoonViT with TP/DP support."""
@@ -355,23 +392,33 @@ class MoonViTEncoderLayer(nn.Module):
*,
activation=F.gelu,
attn_bias: bool = False,
qkv_hidden_size: int | None = None,
norm_type: str = "layernorm",
mlp_type: str = "mlp2",
linear_bias: bool = True,
):
super().__init__()
self.use_data_parallel = is_vit_use_data_parallel(num_heads)
self.num_heads = num_heads
self.hidden_dim = hidden_dim
self.hidden_size_per_attention_head = self.hidden_dim // self.num_heads
self.qkv_hidden_size = (
hidden_dim if qkv_hidden_size is None else qkv_hidden_size
)
self.hidden_size_per_attention_head = self.qkv_hidden_size // self.num_heads
self.tp_size = (
1 if self.use_data_parallel else get_tensor_model_parallel_world_size()
)
self.num_attention_heads_per_partition = divide(num_heads, self.tp_size)
self.norm0 = nn.LayerNorm(hidden_dim)
self.norm1 = nn.LayerNorm(hidden_dim)
self.norm0 = _make_vision_norm(norm_type, hidden_dim)
self.norm1 = _make_vision_norm(norm_type, hidden_dim)
if mlp_type != "mlp2":
raise NotImplementedError(f"Not support mlp_type: {mlp_type}")
self.mlp = MLP2(
[hidden_dim, mlp_dim, hidden_dim],
activation,
bias=linear_bias,
quant_config=quant_config,
prefix=f"{prefix}.mlp",
use_data_parallel=self.use_data_parallel,
@@ -387,7 +434,7 @@ class MoonViTEncoderLayer(nn.Module):
disable_tp=self.use_data_parallel,
)
self.wo = RowParallelLinear(
hidden_dim,
self.qkv_hidden_size,
hidden_dim,
bias=attn_bias,
quant_config=quant_config,
@@ -493,8 +540,9 @@ class MoonViT3dEncoder(nn.Module):
f'video_attn_type must be "spatial_temporal", got {video_attn_type}'
)
self.video_attn_type = video_attn_type
qkv_hidden_size = block_cfg.get("qkv_hidden_size") or block_cfg["hidden_dim"]
self.rope_2d = Rope2DPosEmbRepeated(
block_cfg["hidden_dim"] // block_cfg["num_heads"], 512, 512
qkv_hidden_size // block_cfg["num_heads"], 512, 512
)
self.blocks = nn.ModuleList(
[
@@ -506,13 +554,17 @@ class MoonViT3dEncoder(nn.Module):
for layer_idx in range(num_layers)
]
)
self.final_layernorm = nn.LayerNorm(hidden_dim)
self.final_layernorm = _make_vision_norm(
block_cfg.get("norm_type", "layernorm"), hidden_dim
)
def prepare_encoder_metadata(
self,
grid_thw_list: list[list[int]],
*,
device: torch.device,
max_batch_size: int | None = None,
max_seqlen_override: int | None = None,
) -> dict[str, torch.Tensor | None]:
metadata: dict[str, torch.Tensor | None] = {}
metadata["rope_freqs_cis"] = self.rope_2d.get_freqs_cis(
@@ -524,15 +576,30 @@ class MoonViT3dEncoder(nn.Module):
cu_seqlens = np.concatenate(
[np.zeros(1, dtype=np.int32), lengths.cumsum(dtype=np.int32)]
)
if max_batch_size is not None:
num_seqs = len(cu_seqlens) - 1
if num_seqs < max_batch_size:
cu_seqlens = np.concatenate(
[
cu_seqlens,
np.full(
max_batch_size - num_seqs,
cu_seqlens[-1],
dtype=np.int32,
),
]
)
attn_backend = self.blocks[0].attn.attn_backend
metadata["sequence_lengths"] = MMEncoderAttention.maybe_compute_seq_lens(
attn_backend, cu_seqlens, device
)
metadata["max_seqlen"] = torch.tensor(
MMEncoderAttention.compute_max_seqlen(attn_backend, cu_seqlens),
dtype=torch.int32,
max_seqlen = (
max_seqlen_override
if max_seqlen_override is not None
else MMEncoderAttention.compute_max_seqlen(attn_backend, cu_seqlens)
)
metadata["max_seqlen"] = torch.tensor(max_seqlen, dtype=torch.int32)
metadata["cu_seqlens"] = MMEncoderAttention.maybe_recompute_cu_seqlens(
attn_backend,
cu_seqlens,
@@ -545,11 +612,12 @@ class MoonViT3dEncoder(nn.Module):
def forward(
self,
hidden_states: torch.Tensor,
grid_thws: torch.Tensor | list[list[int]],
grid_thws: torch.Tensor | list[list[int]] | None,
*,
encoder_metadata: dict[str, torch.Tensor | None] | None = None,
) -> torch.Tensor:
if encoder_metadata is None:
assert grid_thws is not None
grid_thw_list = (
grid_thws if isinstance(grid_thws, list) else grid_thws.tolist()
)
@@ -604,6 +672,34 @@ def tpool_patch_merger(
return outputs
def build_image_merge_gather_idx(
grid_thws: list[list[int]] | list[tuple[int, int, int]],
merge_kernel_size: tuple[int, int],
) -> np.ndarray:
"""Build packed spatial-merge indices for image-only CUDA graphs."""
kh, kw = merge_kernel_size
parts: list[np.ndarray] = []
offset = 0
for t, h, w in grid_thws:
if t != 1:
raise ValueError("Image encoder CUDA graphs require grid T == 1")
idx = np.arange(h * w, dtype=np.int64).reshape(h, w)
idx = idx.reshape(h // kh, kh, w // kw, kw)
parts.append(idx.transpose(0, 2, 1, 3).reshape(-1, kh * kw) + offset)
offset += h * w
if not parts:
return np.empty((0, kh * kw), dtype=np.int64)
return np.concatenate(parts)
def tpool_patch_merger_packed(
x: torch.Tensor,
merge_gather_idx: torch.Tensor,
) -> torch.Tensor:
"""Apply the image-only spatial merge using precomputed tensor indices."""
return x[merge_gather_idx]
class MoonViT3dPretrainedModel(nn.Module):
"""Main vision tower model.
@@ -630,6 +726,10 @@ class MoonViT3dPretrainedModel(nn.Module):
pos_emb_width=config.init_pos_emb_width,
pos_emb_time=config.init_pos_emb_time,
pos_emb_type=config.pos_emb_type,
patch_embed_proj_bias=getattr(config, "patch_embed_proj_bias", True),
pos_emb_interpolation_mode=getattr(
config, "pos_emb_interpolation_mode", "bicubic"
),
)
self.encoder = MoonViT3dEncoder(
@@ -638,9 +738,15 @@ class MoonViT3dPretrainedModel(nn.Module):
block_cfg={
"num_heads": config.num_attention_heads,
"hidden_dim": config.hidden_size,
"qkv_hidden_size": getattr(config, "qkv_hidden_size", None),
"mlp_dim": config.intermediate_size,
"activation": get_act_fn("gelu_pytorch_tanh"),
"attn_bias": True,
"activation": get_act_fn(
getattr(config, "activation_func", "gelu_pytorch_tanh")
),
"attn_bias": getattr(config, "attn_bias", True),
"norm_type": getattr(config, "norm_type", "layernorm"),
"mlp_type": getattr(config, "mlp_type", "mlp2"),
"linear_bias": getattr(config, "linear_bias", True),
},
video_attn_type=config.video_attn_type,
quant_config=quant_config,
@@ -650,7 +756,7 @@ class MoonViT3dPretrainedModel(nn.Module):
def forward(
self,
pixel_values: torch.Tensor,
grid_thws: torch.Tensor | list[list[int]],
grid_thws: torch.Tensor | list[list[int]] | None,
*,
encoder_metadata: dict[str, torch.Tensor | None] | None = None,
) -> torch.Tensor:
@@ -662,6 +768,22 @@ class MoonViT3dPretrainedModel(nn.Module):
Returns:
torch.Tensor: The output tokens.
"""
if encoder_metadata is not None and "pos_embeds" in encoder_metadata:
hidden_states = self.patch_embed(
pixel_values,
None,
pos_embeds=encoder_metadata["pos_embeds"],
)
hidden_states = self.encoder(
hidden_states,
None,
encoder_metadata=encoder_metadata,
)
merge_gather_idx = encoder_metadata["merge_gather_idx"]
assert merge_gather_idx is not None
return tpool_patch_merger_packed(hidden_states, merge_gather_idx)
assert grid_thws is not None
grid_thw_list = grid_thws if isinstance(grid_thws, list) else grid_thws.tolist()
if encoder_metadata is None:
encoder_metadata = self.encoder.prepare_encoder_metadata(
@@ -685,13 +807,40 @@ class MoonViT3dPretrainedModel(nn.Module):
return hidden_states
def prepare_encoder_cudagraph_metadata(
self,
grid_thw_list: list[list[int]],
*,
max_batch_size: int,
max_seqlen_override: int | None = None,
device: torch.device,
) -> dict[str, torch.Tensor | None]:
"""Precompute fixed-buffer metadata for image encoder CUDA graphs."""
grid_thw_list = [list(map(int, grid)) for grid in grid_thw_list]
metadata = self.encoder.prepare_encoder_metadata(
grid_thw_list,
device=device,
max_batch_size=max_batch_size,
max_seqlen_override=max_seqlen_override,
)
metadata["pos_embeds"] = self.patch_embed.pos_emb.get_pos_embeds(
grid_thw_list
).to(device=device)
merge_gather_idx = build_image_merge_gather_idx(
grid_thw_list, self.merge_kernel_size
)
metadata["merge_gather_idx"] = torch.from_numpy(merge_gather_idx).to(
device=device, non_blocking=True
)
return metadata
@torch.inference_mode()
def mm_projector_forward(mm_projector: torch.nn.Module, vt_output: list[torch.Tensor]):
"""Apply MM projector to vision tower outputs."""
num_embedding_list = [x.shape[0] for x in vt_output]
batched = torch.cat(vt_output, dim=0)
projector_dtype = mm_projector.pre_norm.weight.dtype
projector_dtype = next(mm_projector.parameters()).dtype
if batched.dtype != projector_dtype:
batched = batched.to(projector_dtype)
proj_out = mm_projector(batched)
@@ -747,11 +896,34 @@ class KimiK25MultiModalProjector(nn.Module):
):
super().__init__()
self.use_data_parallel = use_data_parallel
self.mm_projector_type = getattr(config, "mm_projector_type", "patchmerger")
# Hidden size after patch merging
merge_h, merge_w = config.merge_kernel_size
self.hidden_size = config.hidden_size * merge_h * merge_w
if self.mm_projector_type == "patchmergerv2":
self.linear_1 = ReplicatedLinear(
self.hidden_size,
self.hidden_size,
bias=False,
quant_config=quant_config,
prefix=f"{prefix}.linear_1",
)
self.linear_2 = ReplicatedLinear(
self.hidden_size,
getattr(config, "text_hidden_size", config.mm_hidden_size),
bias=False,
quant_config=quant_config,
prefix=f"{prefix}.linear_2",
)
self.post_norm = torch.nn.RMSNorm(
getattr(config, "text_hidden_size", config.mm_hidden_size),
eps=config.projector_ln_eps,
)
self.act = GELUActivation()
return
self.pre_norm = torch.nn.LayerNorm(config.hidden_size, eps=1e-5)
self.linear_1 = ReplicatedLinear(
self.hidden_size,
@@ -770,6 +942,13 @@ class KimiK25MultiModalProjector(nn.Module):
self.act = GELUActivation()
def forward(self, image_features: torch.Tensor) -> torch.Tensor:
if self.mm_projector_type == "patchmergerv2":
hidden_states = image_features.view(image_features.shape[0], -1)
hidden_states, _ = self.linear_1(hidden_states)
hidden_states = self.act(hidden_states)
hidden_states, _ = self.linear_2(hidden_states)
return self.post_norm(hidden_states)
hidden_states = self.pre_norm(image_features).view(-1, self.hidden_size)
hidden_states, _ = self.linear_1(hidden_states)
hidden_states = self.act(hidden_states)
-646
View File
@@ -1,646 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Iterable
import torch
from torch import nn
from vllm.compilation.decorators import support_torch_compile
from vllm.config import CacheConfig, VllmConfig
from vllm.distributed import (
get_pp_group,
get_tensor_model_parallel_world_size,
)
from vllm.logger import init_logger
from vllm.model_executor.layers.activation import SiluAndMul
from vllm.model_executor.layers.fused_moe import (
FusedMoE,
fused_moe_make_expert_params_mapping,
)
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.linear import (
ColumnParallelLinear,
MergedColumnParallelLinear,
ReplicatedLinear,
RowParallelLinear,
)
from vllm.model_executor.layers.logits_processor import LogitsProcessor
from vllm.model_executor.layers.mamba.gdn.kimi_gdn_linear_attn import (
KimiGatedDeltaNetAttention,
)
from vllm.model_executor.layers.mamba.mamba_utils import (
MambaStateCopyFunc,
MambaStateCopyFuncCalculator,
MambaStateDtypeCalculator,
MambaStateShapeCalculator,
)
from vllm.model_executor.layers.mla import MLAModules, MultiHeadLatentAttentionWrapper
from vllm.model_executor.layers.quantization.base_config import QuantizationConfig
from vllm.model_executor.layers.vocab_parallel_embedding import (
ParallelLMHead,
VocabParallelEmbedding,
)
from vllm.model_executor.model_loader.weight_utils import (
default_weight_loader,
maybe_remap_kv_scale_name,
)
from vllm.sequence import IntermediateTensors
from vllm.transformers_utils.configs.kimi_linear import KimiLinearConfig
from .interfaces import HasInnerState, IsHybrid, MixtureOfExperts, SupportsPP
from .utils import (
AutoWeightsLoader,
PPMissingLayer,
get_spec_layer_idx_from_weight_name,
is_pp_missing_parameter,
make_layers,
maybe_prefix,
)
logger = init_logger(__name__)
class KimiMLP(nn.Module):
def __init__(
self,
hidden_size: int,
intermediate_size: int,
hidden_act: str,
quant_config: QuantizationConfig | None = None,
reduce_results: bool = True,
prefix: str = "",
) -> None:
super().__init__()
self.gate_up_proj = MergedColumnParallelLinear(
hidden_size,
[intermediate_size] * 2,
bias=False,
quant_config=quant_config,
prefix=f"{prefix}.gate_up_proj",
)
self.down_proj = RowParallelLinear(
intermediate_size,
hidden_size,
bias=False,
quant_config=quant_config,
reduce_results=reduce_results,
prefix=f"{prefix}.down_proj",
)
if hidden_act != "silu":
raise ValueError(
f"Unsupported activation: {hidden_act}. Only silu is supported for now."
)
self.act_fn = SiluAndMul()
def forward(self, x):
gate_up, _ = self.gate_up_proj(x)
x = self.act_fn(gate_up)
x, _ = self.down_proj(x)
return x
class KimiMoE(nn.Module):
def __init__(
self,
config: KimiLinearConfig,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
layer_idx: int = 0,
):
super().__init__()
hidden_size = config.hidden_size
intermediate_size = config.intermediate_size
moe_intermediate_size = config.moe_intermediate_size
num_experts = config.num_experts
moe_renormalize = config.moe_renormalize
self.tp_size = get_tensor_model_parallel_world_size()
self.routed_scaling_factor = config.routed_scaling_factor
self.num_shared_experts = config.num_shared_experts
self.layer_idx = layer_idx
if config.hidden_act != "silu":
raise ValueError(
f"Unsupported activation: {config.hidden_act}. "
"Only silu is supported for now."
)
# Gate always runs at half / full precision for now.
self.gate = ReplicatedLinear(
hidden_size,
num_experts,
bias=False,
quant_config=None,
prefix=f"{prefix}.gate",
)
self.gate.e_score_correction_bias = nn.Parameter(torch.empty(num_experts))
if self.num_shared_experts is not None:
intermediate_size = moe_intermediate_size * self.num_shared_experts
self.shared_experts = KimiMLP(
hidden_size=config.hidden_size,
intermediate_size=intermediate_size,
hidden_act=config.hidden_act,
quant_config=quant_config,
reduce_results=False,
prefix=f"{prefix}.shared_experts",
)
else:
self.shared_experts = None
self.experts = FusedMoE(
shared_experts=self.shared_experts,
num_experts=num_experts,
top_k=config.num_experts_per_token,
hidden_size=hidden_size,
intermediate_size=moe_intermediate_size,
renormalize=moe_renormalize,
quant_config=quant_config,
use_grouped_topk=config.use_grouped_topk,
num_expert_group=config.num_expert_group,
topk_group=config.topk_group,
prefix=f"{prefix}.experts",
scoring_func=config.moe_router_activation_func,
e_score_correction_bias=self.gate.e_score_correction_bias,
routed_scaling_factor=self.routed_scaling_factor,
)
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
num_tokens, hidden_size = hidden_states.shape
hidden_states = hidden_states.view(-1, hidden_size)
router_logits, _ = self.gate(hidden_states)
final_hidden_states = self.experts(
hidden_states=hidden_states, router_logits=router_logits
)
return final_hidden_states.view(num_tokens, hidden_size)
class KimiMLAAttention(nn.Module):
"""
Main reference: DeepseekV2 vllm Implementation
"""
def __init__(
self,
config: KimiLinearConfig,
hidden_size: int,
num_heads: int,
qk_nope_head_dim: int,
qk_rope_head_dim: int,
v_head_dim: int,
q_lora_rank: int | None,
kv_lora_rank: int,
use_nope: bool = False,
cache_config: CacheConfig | None = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
**kwargs,
) -> None:
super().__init__()
self.hidden_size = hidden_size
self.qk_nope_head_dim = qk_nope_head_dim
self.qk_rope_head_dim = qk_rope_head_dim
self.qk_head_dim = qk_nope_head_dim + qk_rope_head_dim
self.v_head_dim = v_head_dim
self.q_lora_rank = q_lora_rank
self.kv_lora_rank = kv_lora_rank
self.num_heads = num_heads
tp_size = get_tensor_model_parallel_world_size()
self.num_local_heads = num_heads // tp_size
self.scaling = self.qk_head_dim**-0.5
self.use_nope = use_nope
assert self.use_nope is True
assert self.q_lora_rank is None
assert num_heads % tp_size == 0
self.kv_a_proj_with_mqa = ReplicatedLinear(
self.hidden_size,
self.kv_lora_rank + self.qk_rope_head_dim,
bias=False,
quant_config=quant_config,
prefix=f"{prefix}.kv_a_proj_with_mqa",
)
self.q_proj = ColumnParallelLinear(
self.hidden_size,
self.num_heads * self.qk_head_dim,
bias=False,
quant_config=quant_config,
prefix=f"{prefix}.q_proj",
)
self.kv_a_layernorm = RMSNorm(
self.kv_lora_rank,
eps=config.rms_norm_eps,
)
self.kv_b_proj = ColumnParallelLinear(
self.kv_lora_rank,
self.num_heads * (self.qk_nope_head_dim + self.v_head_dim),
bias=False,
quant_config=quant_config,
prefix=f"{prefix}.kv_b_proj",
)
self.o_proj = RowParallelLinear(
self.num_heads * self.v_head_dim,
self.hidden_size,
bias=False,
quant_config=quant_config,
prefix=f"{prefix}.o_proj",
)
mla_modules = MLAModules(
kv_a_layernorm=self.kv_a_layernorm,
kv_b_proj=self.kv_b_proj,
rotary_emb=None,
o_proj=self.o_proj,
fused_qkv_a_proj=None,
kv_a_proj_with_mqa=self.kv_a_proj_with_mqa,
q_a_layernorm=None,
q_b_proj=None,
q_proj=self.q_proj,
indexer=None,
is_sparse=False,
topk_indices_buffer=None,
)
self.mla_attn = MultiHeadLatentAttentionWrapper(
self.hidden_size,
self.num_local_heads,
self.scaling,
self.qk_nope_head_dim,
self.qk_rope_head_dim,
self.v_head_dim,
self.q_lora_rank,
self.kv_lora_rank,
mla_modules,
cache_config,
quant_config,
prefix,
)
def forward(
self,
positions: torch.Tensor,
hidden_states: torch.Tensor,
output: torch.Tensor,
) -> None:
output[:] = self.mla_attn(positions, hidden_states)
class KimiDecoderLayer(nn.Module):
def __init__(
self,
config: KimiLinearConfig,
vllm_config: VllmConfig,
prefix: str = "",
) -> None:
super().__init__()
self.hidden_size = config.hidden_size
self.is_moe = config.is_moe
layer_idx = int(prefix.rsplit(".", 1)[1])
model_config = vllm_config.model_config
cache_config = vllm_config.cache_config
quant_config = vllm_config.quant_config
if config.is_kda_layer(layer_idx):
self.self_attn = KimiGatedDeltaNetAttention(
config,
vllm_config,
prefix=f"{prefix}.self_attn",
)
else:
self.self_attn = KimiMLAAttention(
layer_idx=layer_idx,
hidden_size=self.hidden_size,
num_heads=config.num_attention_heads,
quant_config=quant_config,
cache_config=cache_config,
model_config=model_config,
prefix=f"{prefix}.self_attn",
config=config,
qk_nope_head_dim=config.qk_nope_head_dim,
qk_rope_head_dim=config.qk_rope_head_dim,
v_head_dim=config.v_head_dim,
q_lora_rank=config.q_lora_rank,
kv_lora_rank=config.kv_lora_rank,
use_nope=config.mla_use_nope,
)
if (
self.is_moe
and config.num_experts is not None
and layer_idx >= config.first_k_dense_replace
and layer_idx % config.moe_layer_freq == 0
):
self.block_sparse_moe = KimiMoE(
config=config,
quant_config=quant_config,
prefix=f"{prefix}.block_sparse_moe",
)
self.mlp = self.block_sparse_moe
else:
self.mlp = KimiMLP(
hidden_size=self.hidden_size,
intermediate_size=config.intermediate_size,
hidden_act=config.hidden_act,
quant_config=quant_config,
prefix=f"{prefix}.mlp",
)
self.input_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
self.post_attention_layernorm = RMSNorm(
config.hidden_size, eps=config.rms_norm_eps
)
def forward(
self,
positions: torch.Tensor,
hidden_states: torch.Tensor,
residual: torch.Tensor | None,
**kwargs,
) -> tuple[torch.Tensor, torch.Tensor]:
# Self Attention
if residual is None:
residual = hidden_states
hidden_states = self.input_layernorm(hidden_states)
else:
hidden_states, residual = self.input_layernorm(hidden_states, residual)
attn_output = torch.empty_like(hidden_states)
self.self_attn(
hidden_states=hidden_states,
positions=positions,
output=attn_output,
)
hidden_states = attn_output
# Fully Connected
hidden_states, residual = self.post_attention_layernorm(hidden_states, residual)
hidden_states = self.mlp(hidden_states)
return hidden_states, residual
@support_torch_compile
class KimiLinearModel(nn.Module):
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
super().__init__()
config = vllm_config.model_config.hf_text_config
self.config = config
self.vocab_size = config.vocab_size
if get_pp_group().is_first_rank:
self.embed_tokens = VocabParallelEmbedding(
config.vocab_size,
config.hidden_size,
prefix=f"{prefix}.embed_tokens",
)
else:
self.embed_tokens = PPMissingLayer()
def get_layer(prefix: str):
return KimiDecoderLayer(
config,
vllm_config,
prefix,
)
self.start_layer, self.end_layer, self.layers = make_layers(
config.num_hidden_layers,
get_layer,
prefix=f"{prefix}.layers",
)
if get_pp_group().is_last_rank:
self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
else:
self.norm = PPMissingLayer()
world_size = get_tensor_model_parallel_world_size()
assert config.num_attention_heads % world_size == 0, (
"num_attention_heads must be divisible by world_size"
)
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
return self.embed_tokens(input_ids)
def forward(
self,
input_ids: torch.Tensor | None,
positions: torch.Tensor,
intermediate_tensors: IntermediateTensors | None,
inputs_embeds: torch.Tensor | None = None,
**kwargs,
) -> torch.Tensor:
if get_pp_group().is_first_rank:
if inputs_embeds is not None:
hidden_states = inputs_embeds
else:
hidden_states = self.embed_input_ids(input_ids)
residual = None
else:
assert intermediate_tensors is not None
hidden_states = intermediate_tensors["hidden_states"]
residual = intermediate_tensors["residual"]
for _, layer in enumerate(self.layers[self.start_layer : self.end_layer]):
hidden_states, residual = layer(
positions=positions,
hidden_states=hidden_states,
residual=residual,
)
if not get_pp_group().is_last_rank:
return IntermediateTensors(
{"hidden_states": hidden_states, "residual": residual}
)
hidden_states, _ = self.norm(hidden_states, residual)
return hidden_states
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
stacked_params_mapping = [
# (param_name, shard_name, shard_id)
(".gate_up_proj", ".gate_proj", 0),
(".gate_up_proj", ".up_proj", 1),
]
if self.config.is_moe:
# Params for weights, fp8 weight scales, fp8 activation scales
# (param_name, weight_name, expert_id, shard_id)
expert_params_mapping = fused_moe_make_expert_params_mapping(
self,
ckpt_gate_proj_name="w1",
ckpt_down_proj_name="w2",
ckpt_up_proj_name="w3",
num_experts=self.config.num_experts,
)
else:
expert_params_mapping = []
params_dict = dict(self.named_parameters())
loaded_params: set[str] = set()
for args in weights:
name, loaded_weight = args[:2]
kwargs = args[2] if len(args) > 2 else {}
if "rotary_emb.inv_freq" in name:
continue
spec_layer = get_spec_layer_idx_from_weight_name(self.config, name)
if spec_layer is not None:
continue # skip spec decode layers for main model
if "rotary_emb.cos_cached" in name or "rotary_emb.sin_cached" in name:
# Models trained using ColossalAI may include these tensors in
# the checkpoint. Skip them.
continue
for param_name, weight_name, shard_id in stacked_params_mapping:
if weight_name not in name:
continue
# We have mlp.experts[0].gate_proj in the checkpoint.
# Since we handle the experts below in expert_params_mapping,
# we need to skip here BEFORE we update the name, otherwise
# name will be updated to mlp.experts[0].gate_up_proj, which
# will then be updated below in expert_params_mapping
# for mlp.experts[0].gate_gate_up_proj, which breaks load.
if ("mlp.experts." in name) and name not in params_dict:
continue
name = name.replace(weight_name, param_name)
# Skip loading extra bias for GPTQ models.
if name.endswith(".bias") and name not in params_dict:
continue
if is_pp_missing_parameter(name, self):
continue
param = params_dict[name]
weight_loader = param.weight_loader
weight_loader(param, loaded_weight, shard_id)
break
else:
for idx, (param_name, weight_name, expert_id, shard_id) in enumerate(
expert_params_mapping
):
if weight_name not in name:
continue
name = name.replace(weight_name, param_name)
if is_pp_missing_parameter(name, self):
continue
param = params_dict[name]
weight_loader = param.weight_loader
weight_loader(
param,
loaded_weight,
name,
expert_id=expert_id,
shard_id=shard_id,
)
break
else:
# Skip loading extra bias for GPTQ models.
if (
name.endswith(".bias")
and name not in params_dict
and not self.config.is_linear_attn
): # noqa: E501
continue
# Remapping the name of FP8 kv-scale.
name = maybe_remap_kv_scale_name(name, params_dict)
if name is None:
continue
if is_pp_missing_parameter(name, self):
continue
param = params_dict[name]
weight_loader = getattr(
param, "weight_loader", default_weight_loader
)
weight_loader(param, loaded_weight, **kwargs)
loaded_params.add(name)
return loaded_params
class KimiLinearForCausalLM(
nn.Module, HasInnerState, SupportsPP, MixtureOfExperts, IsHybrid
):
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
super().__init__()
self.model_config = vllm_config.model_config
self.vllm_config = vllm_config
self.config = self.model_config.hf_config
quant_config = vllm_config.quant_config
self.quant_config = quant_config
self.model = KimiLinearModel(
vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model")
)
if get_pp_group().is_last_rank:
self.lm_head = ParallelLMHead(
self.config.vocab_size,
self.config.hidden_size,
quant_config=quant_config,
prefix=maybe_prefix(prefix, "lm_head"),
)
else:
self.lm_head = PPMissingLayer()
logit_scale = getattr(self.config, "logit_scale", 1.0)
self.logits_processor = LogitsProcessor(
self.config.vocab_size, scale=logit_scale
)
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
return self.model.embed_input_ids(input_ids)
def forward(
self,
input_ids: torch.Tensor | None,
positions: torch.Tensor,
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
**kwargs,
) -> torch.Tensor | IntermediateTensors:
hidden_states = self.model(
input_ids, positions, intermediate_tensors, inputs_embeds, **kwargs
)
return hidden_states
@classmethod
def get_mamba_state_dtype_from_config(
cls,
vllm_config: "VllmConfig",
) -> tuple[torch.dtype, torch.dtype]:
return MambaStateDtypeCalculator.kda_state_dtype(
vllm_config.model_config.dtype, vllm_config.cache_config.mamba_cache_dtype
)
@classmethod
def get_mamba_state_shape_from_config(
cls, vllm_config: "VllmConfig"
) -> tuple[tuple[int, ...], tuple[int, ...]]:
parallel_config = vllm_config.parallel_config
hf_config = vllm_config.model_config.hf_config
tp_size = parallel_config.tensor_parallel_size
num_spec = (
vllm_config.speculative_config.num_speculative_tokens
if vllm_config.speculative_config
else 0
)
return MambaStateShapeCalculator.kda_state_shape(
tp_size,
hf_config.linear_attn_config["num_heads"],
hf_config.linear_attn_config["head_dim"],
conv_kernel_size=hf_config.linear_attn_config["short_conv_kernel_size"],
num_spec=num_spec,
)
@classmethod
def get_mamba_state_copy_func(
cls,
) -> tuple[MambaStateCopyFunc, MambaStateCopyFunc]:
return MambaStateCopyFuncCalculator.kda_state_copy_func()
def compute_logits(
self,
hidden_states: torch.Tensor,
) -> torch.Tensor | None:
return self.logits_processor(self.lm_head, hidden_states)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
loader = AutoWeightsLoader(
self,
skip_prefixes=(["lm_head."] if self.config.tie_word_embeddings else None),
)
return loader.load_weights(weights)
+13 -1
View File
@@ -137,7 +137,10 @@ _TEXT_GENERATION_MODELS = {
"IQuestLoopCoderForCausalLM": ("iquest_loopcoder", "IQuestLoopCoderForCausalLM"),
"Jais2ForCausalLM": ("jais2", "Jais2ForCausalLM"),
"JambaForCausalLM": ("jamba", "JambaForCausalLM"),
"KimiLinearForCausalLM": ("kimi_linear", "KimiLinearForCausalLM"),
"KimiLinearForCausalLM": (
"vllm.models.kimi_k3",
"KimiLinearForCausalLM",
),
"Lfm2ForCausalLM": ("lfm2", "Lfm2ForCausalLM"),
"Lfm2MoeForCausalLM": ("lfm2_moe", "Lfm2MoeForCausalLM"),
"LagunaForCausalLM": ("laguna", "LagunaForCausalLM"),
@@ -460,6 +463,10 @@ _MULTIMODAL_MODELS = {
),
"KimiVLForConditionalGeneration": ("kimi_vl", "KimiVLForConditionalGeneration"),
"KimiK25ForConditionalGeneration": ("kimi_k25", "KimiK25ForConditionalGeneration"),
"KimiK3ForConditionalGeneration": (
"vllm.models.kimi_k3",
"KimiK3ForConditionalGeneration",
),
"MoonshotKimiaForCausalLM": ("kimi_audio", "KimiAudioForConditionalGeneration"),
"MossTranscribeDiarizeForConditionalGeneration": (
"moss_transcribe_diarize",
@@ -609,6 +616,10 @@ _SPECULATIVE_DECODING_MODELS = {
"DFlashDraftModel": ("qwen3_dflash", "DFlashQwen3ForCausalLM"),
"DSparkDraftModel": ("vllm.models.deepseek_v4", "DSparkDeepseekV4ForCausalLM"),
"Qwen3DSparkModel": ("qwen3_dspark", "Qwen3DSparkForCausalLM"),
"K3DSparkModel": (
"vllm.models.kimi_k3.nvidia.dspark_mla",
"K3DSparkForCausalLM",
),
"DFlashLagunaForCausalLM": ("laguna_dflash", "DFlashLagunaForCausalLM"),
"Gemma4DSparkModel": ("gemma4_dspark", "Gemma4DSparkForCausalLM"),
"PEagleDraftModel": ("llama_eagle3", "Eagle3LlamaForCausalLM"),
@@ -649,6 +660,7 @@ _SPECULATIVE_DECODING_MODELS = {
"Qwen3_5MTP": ("qwen3_5_mtp", "Qwen3_5MTP"),
"Qwen3_5MoeMTP": ("qwen3_5_mtp", "Qwen3_5MoeMTP"),
"HYV3MTPModel": ("hy_v3_mtp", "HYV3MTP"),
"KimiK3MTPModel": ("vllm.models.kimi_k3", "KimiK3MTP"),
# Temporarily disabled.
# # TODO(woosuk): Re-enable this once the MLP Speculator is supported in V1.
# "MLPSpeculatorPreTrainedModel": ("mlp_speculator", "MLPSpeculator"),