[New model] Kimi K3 (#50000)
Signed-off-by: zjy0516 <riverclouds.zhu@qq.com> Signed-off-by: Isotr0py <Isotr0py@outlook.com> Signed-off-by: Bugen Zhao <i@bugenzhao.com> Signed-off-by: Jee Jee Li <jeejeelee@inferact.ai> Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai> Signed-off-by: Yongye Zhu <zyy1102000@gmail.com> Signed-off-by: wzhao18 <wzhao18.sz@gmail.com> Co-authored-by: Isotr0py <Isotr0py@outlook.com> Co-authored-by: Thien Tran <gau.nernst@yahoo.com.sg> Co-authored-by: Bugen Zhao <i@bugenzhao.com> Co-authored-by: Yongye Zhu <zyy1102000@gmail.com> Co-authored-by: Ziming Huang <zelda.huanghuang@gmail.com> Co-authored-by: Roger Wang <hey@rogerw.io> Co-authored-by: Isotr0py <mozf@inferact.ai> Co-authored-by: aoshen02 <aoshen02@users.noreply.github.com> Co-authored-by: Woosuk Kwon <woosuk@inferact.ai> Co-authored-by: Jee Jee Li <jeejeelee@inferact.ai> Co-authored-by: aoshen02 <aoshen@inferact.ai> Co-authored-by: Summer Yang <girasoleyang@gmail.com> Co-authored-by: Kevin H. Luu <khluu000@gmail.com> Co-authored-by: Bowen Wang <abmfy@icloud.com> Co-authored-by: gnovack <novackgm@gmail.com> Co-authored-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai> Co-authored-by: xiaozhoupy <peiyuanzhou1994@gmail.com> Co-authored-by: Roy Wang <yasong.wang@inferact.ai> Co-authored-by: Jeff (Junze) Ma <93145857+majunze2001@users.noreply.github.com> Co-authored-by: Codex <codex@openai.com> Co-authored-by: Wei Zhao <51183510+wzhao18@users.noreply.github.com>
This commit is contained in:
co-authored by
Isotr0py
Thien Tran
Bugen Zhao
Yongye Zhu
Ziming Huang
Roger Wang
Isotr0py
aoshen02
Woosuk Kwon
Jee Jee Li
aoshen02
Summer Yang
Kevin H. Luu
Bowen Wang
gnovack
Nick Hill
Yifan Qiao
xiaozhoupy
Roy Wang
Jeff Ma
Codex
Wei Zhao
parent
0c64be8873
commit
aeeb36b1f1
@@ -339,6 +339,39 @@ class DeepseekV4ForCausalLMConfig(VerifyAndUpdateConfig):
|
||||
)
|
||||
|
||||
|
||||
class KimiK3ForConditionalGenerationConfig(VerifyAndUpdateConfig):
|
||||
"""Route MXFP4-checkpointed Kimi-K3 MoE experts to the MXFP4 interface.
|
||||
|
||||
Kimi-K3 ships its routed experts as compressed-tensors
|
||||
``mxfp4-pack-quantized`` (``quant_method="compressed-tensors"``), which
|
||||
lands them on ``CompressedTensorsW4A4Mxfp4MoEMethod`` and its narrow kernel
|
||||
selection. Rewriting ``quant_method`` to ``"mxfp4"`` selects ``Mxfp4Config``
|
||||
(hence ``Mxfp4MoEMethod``) with its full backend set, while any non-MXFP4
|
||||
checkpoint is left untouched. Covers both the main model and the MTP draft.
|
||||
|
||||
``model_arch_config.quantization_config`` is a separate dict, snapshotted in
|
||||
``ModelConfig.__init__`` before this hook runs, and it is what
|
||||
``_verify_quantization`` reads when resolving the quant method. Patch it
|
||||
alongside the hf configs so the rewrite lands before resolution; otherwise
|
||||
the main model still resolves to compressed-tensors.
|
||||
"""
|
||||
|
||||
@staticmethod
|
||||
def verify_and_update_model_config(model_config: "ModelConfig") -> None:
|
||||
for cfg in (
|
||||
model_config.hf_config,
|
||||
model_config.hf_text_config,
|
||||
model_config.model_arch_config,
|
||||
):
|
||||
quant_config = getattr(cfg, "quantization_config", None)
|
||||
if (
|
||||
isinstance(quant_config, dict)
|
||||
and quant_config.get("quant_method") == "compressed-tensors"
|
||||
and quant_config.get("format") == "mxfp4-pack-quantized"
|
||||
):
|
||||
quant_config["quant_method"] = "mxfp4"
|
||||
|
||||
|
||||
class GptOssForCausalLMConfig(VerifyAndUpdateConfig):
|
||||
@staticmethod
|
||||
def verify_and_update_model_config(model_config: "ModelConfig") -> None:
|
||||
@@ -884,6 +917,8 @@ MODELS_CONFIG_MAP: dict[str, type[VerifyAndUpdateConfig]] = {
|
||||
"JambaForSequenceClassification": JambaForSequenceClassificationConfig,
|
||||
"JinaForRanking": JinaForRankingConfig,
|
||||
"JinaVLForRanking": JinaVLForSequenceClassificationConfig,
|
||||
"KimiK3ForConditionalGeneration": KimiK3ForConditionalGenerationConfig,
|
||||
"KimiK3MTPModel": KimiK3ForConditionalGenerationConfig,
|
||||
"LlamaBidirectionalForSequenceClassification": LlamaBidirectionalConfig,
|
||||
"LlamaBidirectionalModel": LlamaBidirectionalConfig,
|
||||
"LlamaNemotronVLForSequenceClassification": LlamaNemotronVLConfig,
|
||||
|
||||
@@ -983,6 +983,7 @@ class DeepseekV2MLAAttention(nn.Module):
|
||||
topk_indices_buffer: torch.Tensor | None = None,
|
||||
input_size: int | None = None,
|
||||
reduce_results: bool = True,
|
||||
non_causal_multi_token_decode: bool = False,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.hidden_size = hidden_size
|
||||
@@ -1178,6 +1179,7 @@ class DeepseekV2MLAAttention(nn.Module):
|
||||
# the V1 proposer. A frozen True would leave the draft reading a
|
||||
# never-written topk buffer.
|
||||
skip_topk=_skip_topk and not is_mtp_layer,
|
||||
non_causal_multi_token_decode=non_causal_multi_token_decode,
|
||||
# Do not skip scoring for MTP layers: their top-k buffer may be
|
||||
# reused by later draft iterations through index sharing.
|
||||
allow_short_prefill_indexer_scoring_skip=not is_mtp_layer,
|
||||
|
||||
@@ -1407,7 +1407,7 @@ class EagleModelMixin:
|
||||
aux_hidden_states: list[torch.Tensor],
|
||||
layer_idx: int,
|
||||
hidden_states: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
residual: torch.Tensor | None,
|
||||
) -> list[torch.Tensor]:
|
||||
if layer_idx in self.aux_hidden_state_layers:
|
||||
value = hidden_states + residual if residual is not None else hidden_states
|
||||
|
||||
@@ -34,6 +34,7 @@ from vllm.model_executor.models.vision import (
|
||||
is_vit_use_data_parallel,
|
||||
run_dp_sharded_mrope_vision_model,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.transformers_utils.configs.kimi_k25 import KimiK25VisionConfig
|
||||
|
||||
logger = init_logger(__name__)
|
||||
@@ -154,9 +155,7 @@ class Learnable2DInterpPosEmbDivided_fixed(nn.Module):
|
||||
def reset_parameters(self):
|
||||
nn.init.normal_(self.weight)
|
||||
|
||||
def forward(
|
||||
self, x: torch.Tensor, grid_thws: torch.Tensor | list[list[int]]
|
||||
) -> torch.Tensor:
|
||||
def get_pos_embeds(self, grid_thws: torch.Tensor | list[list[int]]) -> torch.Tensor:
|
||||
pos_embs = []
|
||||
grid_thw_list = grid_thws if isinstance(grid_thws, list) else grid_thws.tolist()
|
||||
for t, h, w in grid_thw_list:
|
||||
@@ -179,8 +178,12 @@ class Learnable2DInterpPosEmbDivided_fixed(nn.Module):
|
||||
|
||||
pos_embs.append(pos_emb_3d.reshape(-1, pos_emb_3d.shape[-1]))
|
||||
|
||||
out = x + torch.cat(pos_embs)
|
||||
return out
|
||||
return torch.cat(pos_embs)
|
||||
|
||||
def forward(
|
||||
self, x: torch.Tensor, grid_thws: torch.Tensor | list[list[int]]
|
||||
) -> torch.Tensor:
|
||||
return x + self.get_pos_embeds(grid_thws)
|
||||
|
||||
|
||||
class MoonVision3dPatchEmbed(nn.Module):
|
||||
@@ -195,6 +198,8 @@ class MoonVision3dPatchEmbed(nn.Module):
|
||||
pos_emb_width: int = 14,
|
||||
pos_emb_time: int = 4,
|
||||
pos_emb_type: str = "divided_fixed",
|
||||
patch_embed_proj_bias: bool = True,
|
||||
pos_emb_interpolation_mode: str = "bicubic",
|
||||
):
|
||||
super().__init__()
|
||||
assert isinstance(patch_size, int | Sequence), (
|
||||
@@ -208,7 +213,11 @@ class MoonVision3dPatchEmbed(nn.Module):
|
||||
self.patch_size = patch_size
|
||||
|
||||
self.proj = nn.Conv2d(
|
||||
in_dim, out_dim, kernel_size=patch_size, stride=patch_size
|
||||
in_dim,
|
||||
out_dim,
|
||||
kernel_size=patch_size,
|
||||
stride=patch_size,
|
||||
bias=patch_embed_proj_bias,
|
||||
)
|
||||
|
||||
if pos_emb_type == "divided_fixed":
|
||||
@@ -217,17 +226,37 @@ class MoonVision3dPatchEmbed(nn.Module):
|
||||
width=pos_emb_width,
|
||||
num_frames=pos_emb_time,
|
||||
dim=out_dim,
|
||||
interpolation_mode=pos_emb_interpolation_mode,
|
||||
)
|
||||
else:
|
||||
raise NotImplementedError(f"Not support pos_emb_type: {pos_emb_type}")
|
||||
|
||||
def forward(
|
||||
self, x: torch.Tensor, grid_thws: torch.Tensor | list[list[int]]
|
||||
self,
|
||||
x: torch.Tensor,
|
||||
grid_thws: torch.Tensor | list[list[int]] | None,
|
||||
*,
|
||||
pos_embeds: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
x = self.proj(x).view(x.size(0), -1)
|
||||
# apply positional embedding
|
||||
x = self.pos_emb(x, grid_thws)
|
||||
return x
|
||||
x = self._proj(x).view(x.size(0), -1)
|
||||
if pos_embeds is not None:
|
||||
return x + pos_embeds
|
||||
assert grid_thws is not None
|
||||
return self.pos_emb(x, grid_thws)
|
||||
|
||||
def _proj(self, x: torch.Tensor) -> torch.Tensor:
|
||||
# MIOpen conv2d intermittently fails under load on ROCm; use aiter Triton.
|
||||
if current_platform.is_rocm() and x.dtype in (torch.float16, torch.bfloat16):
|
||||
from aiter.ops.triton.conv.conv2d import conv2d
|
||||
|
||||
return conv2d(
|
||||
x,
|
||||
self.proj.weight,
|
||||
self.proj.bias,
|
||||
stride=self.patch_size,
|
||||
layout="nchw",
|
||||
)
|
||||
return self.proj(x)
|
||||
|
||||
|
||||
class Rope2DPosEmbRepeated(nn.Module):
|
||||
@@ -342,6 +371,14 @@ class MLP2(nn.Module):
|
||||
return x
|
||||
|
||||
|
||||
def _make_vision_norm(norm_type: str, hidden_dim: int) -> nn.Module:
|
||||
if norm_type == "layernorm":
|
||||
return nn.LayerNorm(hidden_dim)
|
||||
if norm_type == "rmsnorm":
|
||||
return nn.RMSNorm(hidden_dim)
|
||||
raise NotImplementedError(f"Not support norm_type: {norm_type}")
|
||||
|
||||
|
||||
class MoonViTEncoderLayer(nn.Module):
|
||||
"""Single encoder layer for MoonViT with TP/DP support."""
|
||||
|
||||
@@ -355,23 +392,33 @@ class MoonViTEncoderLayer(nn.Module):
|
||||
*,
|
||||
activation=F.gelu,
|
||||
attn_bias: bool = False,
|
||||
qkv_hidden_size: int | None = None,
|
||||
norm_type: str = "layernorm",
|
||||
mlp_type: str = "mlp2",
|
||||
linear_bias: bool = True,
|
||||
):
|
||||
super().__init__()
|
||||
self.use_data_parallel = is_vit_use_data_parallel(num_heads)
|
||||
|
||||
self.num_heads = num_heads
|
||||
self.hidden_dim = hidden_dim
|
||||
self.hidden_size_per_attention_head = self.hidden_dim // self.num_heads
|
||||
self.qkv_hidden_size = (
|
||||
hidden_dim if qkv_hidden_size is None else qkv_hidden_size
|
||||
)
|
||||
self.hidden_size_per_attention_head = self.qkv_hidden_size // self.num_heads
|
||||
self.tp_size = (
|
||||
1 if self.use_data_parallel else get_tensor_model_parallel_world_size()
|
||||
)
|
||||
self.num_attention_heads_per_partition = divide(num_heads, self.tp_size)
|
||||
|
||||
self.norm0 = nn.LayerNorm(hidden_dim)
|
||||
self.norm1 = nn.LayerNorm(hidden_dim)
|
||||
self.norm0 = _make_vision_norm(norm_type, hidden_dim)
|
||||
self.norm1 = _make_vision_norm(norm_type, hidden_dim)
|
||||
if mlp_type != "mlp2":
|
||||
raise NotImplementedError(f"Not support mlp_type: {mlp_type}")
|
||||
self.mlp = MLP2(
|
||||
[hidden_dim, mlp_dim, hidden_dim],
|
||||
activation,
|
||||
bias=linear_bias,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.mlp",
|
||||
use_data_parallel=self.use_data_parallel,
|
||||
@@ -387,7 +434,7 @@ class MoonViTEncoderLayer(nn.Module):
|
||||
disable_tp=self.use_data_parallel,
|
||||
)
|
||||
self.wo = RowParallelLinear(
|
||||
hidden_dim,
|
||||
self.qkv_hidden_size,
|
||||
hidden_dim,
|
||||
bias=attn_bias,
|
||||
quant_config=quant_config,
|
||||
@@ -493,8 +540,9 @@ class MoonViT3dEncoder(nn.Module):
|
||||
f'video_attn_type must be "spatial_temporal", got {video_attn_type}'
|
||||
)
|
||||
self.video_attn_type = video_attn_type
|
||||
qkv_hidden_size = block_cfg.get("qkv_hidden_size") or block_cfg["hidden_dim"]
|
||||
self.rope_2d = Rope2DPosEmbRepeated(
|
||||
block_cfg["hidden_dim"] // block_cfg["num_heads"], 512, 512
|
||||
qkv_hidden_size // block_cfg["num_heads"], 512, 512
|
||||
)
|
||||
self.blocks = nn.ModuleList(
|
||||
[
|
||||
@@ -506,13 +554,17 @@ class MoonViT3dEncoder(nn.Module):
|
||||
for layer_idx in range(num_layers)
|
||||
]
|
||||
)
|
||||
self.final_layernorm = nn.LayerNorm(hidden_dim)
|
||||
self.final_layernorm = _make_vision_norm(
|
||||
block_cfg.get("norm_type", "layernorm"), hidden_dim
|
||||
)
|
||||
|
||||
def prepare_encoder_metadata(
|
||||
self,
|
||||
grid_thw_list: list[list[int]],
|
||||
*,
|
||||
device: torch.device,
|
||||
max_batch_size: int | None = None,
|
||||
max_seqlen_override: int | None = None,
|
||||
) -> dict[str, torch.Tensor | None]:
|
||||
metadata: dict[str, torch.Tensor | None] = {}
|
||||
metadata["rope_freqs_cis"] = self.rope_2d.get_freqs_cis(
|
||||
@@ -524,15 +576,30 @@ class MoonViT3dEncoder(nn.Module):
|
||||
cu_seqlens = np.concatenate(
|
||||
[np.zeros(1, dtype=np.int32), lengths.cumsum(dtype=np.int32)]
|
||||
)
|
||||
if max_batch_size is not None:
|
||||
num_seqs = len(cu_seqlens) - 1
|
||||
if num_seqs < max_batch_size:
|
||||
cu_seqlens = np.concatenate(
|
||||
[
|
||||
cu_seqlens,
|
||||
np.full(
|
||||
max_batch_size - num_seqs,
|
||||
cu_seqlens[-1],
|
||||
dtype=np.int32,
|
||||
),
|
||||
]
|
||||
)
|
||||
|
||||
attn_backend = self.blocks[0].attn.attn_backend
|
||||
metadata["sequence_lengths"] = MMEncoderAttention.maybe_compute_seq_lens(
|
||||
attn_backend, cu_seqlens, device
|
||||
)
|
||||
metadata["max_seqlen"] = torch.tensor(
|
||||
MMEncoderAttention.compute_max_seqlen(attn_backend, cu_seqlens),
|
||||
dtype=torch.int32,
|
||||
max_seqlen = (
|
||||
max_seqlen_override
|
||||
if max_seqlen_override is not None
|
||||
else MMEncoderAttention.compute_max_seqlen(attn_backend, cu_seqlens)
|
||||
)
|
||||
metadata["max_seqlen"] = torch.tensor(max_seqlen, dtype=torch.int32)
|
||||
metadata["cu_seqlens"] = MMEncoderAttention.maybe_recompute_cu_seqlens(
|
||||
attn_backend,
|
||||
cu_seqlens,
|
||||
@@ -545,11 +612,12 @@ class MoonViT3dEncoder(nn.Module):
|
||||
def forward(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
grid_thws: torch.Tensor | list[list[int]],
|
||||
grid_thws: torch.Tensor | list[list[int]] | None,
|
||||
*,
|
||||
encoder_metadata: dict[str, torch.Tensor | None] | None = None,
|
||||
) -> torch.Tensor:
|
||||
if encoder_metadata is None:
|
||||
assert grid_thws is not None
|
||||
grid_thw_list = (
|
||||
grid_thws if isinstance(grid_thws, list) else grid_thws.tolist()
|
||||
)
|
||||
@@ -604,6 +672,34 @@ def tpool_patch_merger(
|
||||
return outputs
|
||||
|
||||
|
||||
def build_image_merge_gather_idx(
|
||||
grid_thws: list[list[int]] | list[tuple[int, int, int]],
|
||||
merge_kernel_size: tuple[int, int],
|
||||
) -> np.ndarray:
|
||||
"""Build packed spatial-merge indices for image-only CUDA graphs."""
|
||||
kh, kw = merge_kernel_size
|
||||
parts: list[np.ndarray] = []
|
||||
offset = 0
|
||||
for t, h, w in grid_thws:
|
||||
if t != 1:
|
||||
raise ValueError("Image encoder CUDA graphs require grid T == 1")
|
||||
idx = np.arange(h * w, dtype=np.int64).reshape(h, w)
|
||||
idx = idx.reshape(h // kh, kh, w // kw, kw)
|
||||
parts.append(idx.transpose(0, 2, 1, 3).reshape(-1, kh * kw) + offset)
|
||||
offset += h * w
|
||||
if not parts:
|
||||
return np.empty((0, kh * kw), dtype=np.int64)
|
||||
return np.concatenate(parts)
|
||||
|
||||
|
||||
def tpool_patch_merger_packed(
|
||||
x: torch.Tensor,
|
||||
merge_gather_idx: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
"""Apply the image-only spatial merge using precomputed tensor indices."""
|
||||
return x[merge_gather_idx]
|
||||
|
||||
|
||||
class MoonViT3dPretrainedModel(nn.Module):
|
||||
"""Main vision tower model.
|
||||
|
||||
@@ -630,6 +726,10 @@ class MoonViT3dPretrainedModel(nn.Module):
|
||||
pos_emb_width=config.init_pos_emb_width,
|
||||
pos_emb_time=config.init_pos_emb_time,
|
||||
pos_emb_type=config.pos_emb_type,
|
||||
patch_embed_proj_bias=getattr(config, "patch_embed_proj_bias", True),
|
||||
pos_emb_interpolation_mode=getattr(
|
||||
config, "pos_emb_interpolation_mode", "bicubic"
|
||||
),
|
||||
)
|
||||
|
||||
self.encoder = MoonViT3dEncoder(
|
||||
@@ -638,9 +738,15 @@ class MoonViT3dPretrainedModel(nn.Module):
|
||||
block_cfg={
|
||||
"num_heads": config.num_attention_heads,
|
||||
"hidden_dim": config.hidden_size,
|
||||
"qkv_hidden_size": getattr(config, "qkv_hidden_size", None),
|
||||
"mlp_dim": config.intermediate_size,
|
||||
"activation": get_act_fn("gelu_pytorch_tanh"),
|
||||
"attn_bias": True,
|
||||
"activation": get_act_fn(
|
||||
getattr(config, "activation_func", "gelu_pytorch_tanh")
|
||||
),
|
||||
"attn_bias": getattr(config, "attn_bias", True),
|
||||
"norm_type": getattr(config, "norm_type", "layernorm"),
|
||||
"mlp_type": getattr(config, "mlp_type", "mlp2"),
|
||||
"linear_bias": getattr(config, "linear_bias", True),
|
||||
},
|
||||
video_attn_type=config.video_attn_type,
|
||||
quant_config=quant_config,
|
||||
@@ -650,7 +756,7 @@ class MoonViT3dPretrainedModel(nn.Module):
|
||||
def forward(
|
||||
self,
|
||||
pixel_values: torch.Tensor,
|
||||
grid_thws: torch.Tensor | list[list[int]],
|
||||
grid_thws: torch.Tensor | list[list[int]] | None,
|
||||
*,
|
||||
encoder_metadata: dict[str, torch.Tensor | None] | None = None,
|
||||
) -> torch.Tensor:
|
||||
@@ -662,6 +768,22 @@ class MoonViT3dPretrainedModel(nn.Module):
|
||||
Returns:
|
||||
torch.Tensor: The output tokens.
|
||||
"""
|
||||
if encoder_metadata is not None and "pos_embeds" in encoder_metadata:
|
||||
hidden_states = self.patch_embed(
|
||||
pixel_values,
|
||||
None,
|
||||
pos_embeds=encoder_metadata["pos_embeds"],
|
||||
)
|
||||
hidden_states = self.encoder(
|
||||
hidden_states,
|
||||
None,
|
||||
encoder_metadata=encoder_metadata,
|
||||
)
|
||||
merge_gather_idx = encoder_metadata["merge_gather_idx"]
|
||||
assert merge_gather_idx is not None
|
||||
return tpool_patch_merger_packed(hidden_states, merge_gather_idx)
|
||||
|
||||
assert grid_thws is not None
|
||||
grid_thw_list = grid_thws if isinstance(grid_thws, list) else grid_thws.tolist()
|
||||
if encoder_metadata is None:
|
||||
encoder_metadata = self.encoder.prepare_encoder_metadata(
|
||||
@@ -685,13 +807,40 @@ class MoonViT3dPretrainedModel(nn.Module):
|
||||
|
||||
return hidden_states
|
||||
|
||||
def prepare_encoder_cudagraph_metadata(
|
||||
self,
|
||||
grid_thw_list: list[list[int]],
|
||||
*,
|
||||
max_batch_size: int,
|
||||
max_seqlen_override: int | None = None,
|
||||
device: torch.device,
|
||||
) -> dict[str, torch.Tensor | None]:
|
||||
"""Precompute fixed-buffer metadata for image encoder CUDA graphs."""
|
||||
grid_thw_list = [list(map(int, grid)) for grid in grid_thw_list]
|
||||
metadata = self.encoder.prepare_encoder_metadata(
|
||||
grid_thw_list,
|
||||
device=device,
|
||||
max_batch_size=max_batch_size,
|
||||
max_seqlen_override=max_seqlen_override,
|
||||
)
|
||||
metadata["pos_embeds"] = self.patch_embed.pos_emb.get_pos_embeds(
|
||||
grid_thw_list
|
||||
).to(device=device)
|
||||
merge_gather_idx = build_image_merge_gather_idx(
|
||||
grid_thw_list, self.merge_kernel_size
|
||||
)
|
||||
metadata["merge_gather_idx"] = torch.from_numpy(merge_gather_idx).to(
|
||||
device=device, non_blocking=True
|
||||
)
|
||||
return metadata
|
||||
|
||||
|
||||
@torch.inference_mode()
|
||||
def mm_projector_forward(mm_projector: torch.nn.Module, vt_output: list[torch.Tensor]):
|
||||
"""Apply MM projector to vision tower outputs."""
|
||||
num_embedding_list = [x.shape[0] for x in vt_output]
|
||||
batched = torch.cat(vt_output, dim=0)
|
||||
projector_dtype = mm_projector.pre_norm.weight.dtype
|
||||
projector_dtype = next(mm_projector.parameters()).dtype
|
||||
if batched.dtype != projector_dtype:
|
||||
batched = batched.to(projector_dtype)
|
||||
proj_out = mm_projector(batched)
|
||||
@@ -747,11 +896,34 @@ class KimiK25MultiModalProjector(nn.Module):
|
||||
):
|
||||
super().__init__()
|
||||
self.use_data_parallel = use_data_parallel
|
||||
self.mm_projector_type = getattr(config, "mm_projector_type", "patchmerger")
|
||||
|
||||
# Hidden size after patch merging
|
||||
merge_h, merge_w = config.merge_kernel_size
|
||||
self.hidden_size = config.hidden_size * merge_h * merge_w
|
||||
|
||||
if self.mm_projector_type == "patchmergerv2":
|
||||
self.linear_1 = ReplicatedLinear(
|
||||
self.hidden_size,
|
||||
self.hidden_size,
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.linear_1",
|
||||
)
|
||||
self.linear_2 = ReplicatedLinear(
|
||||
self.hidden_size,
|
||||
getattr(config, "text_hidden_size", config.mm_hidden_size),
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.linear_2",
|
||||
)
|
||||
self.post_norm = torch.nn.RMSNorm(
|
||||
getattr(config, "text_hidden_size", config.mm_hidden_size),
|
||||
eps=config.projector_ln_eps,
|
||||
)
|
||||
self.act = GELUActivation()
|
||||
return
|
||||
|
||||
self.pre_norm = torch.nn.LayerNorm(config.hidden_size, eps=1e-5)
|
||||
self.linear_1 = ReplicatedLinear(
|
||||
self.hidden_size,
|
||||
@@ -770,6 +942,13 @@ class KimiK25MultiModalProjector(nn.Module):
|
||||
self.act = GELUActivation()
|
||||
|
||||
def forward(self, image_features: torch.Tensor) -> torch.Tensor:
|
||||
if self.mm_projector_type == "patchmergerv2":
|
||||
hidden_states = image_features.view(image_features.shape[0], -1)
|
||||
hidden_states, _ = self.linear_1(hidden_states)
|
||||
hidden_states = self.act(hidden_states)
|
||||
hidden_states, _ = self.linear_2(hidden_states)
|
||||
return self.post_norm(hidden_states)
|
||||
|
||||
hidden_states = self.pre_norm(image_features).view(-1, self.hidden_size)
|
||||
hidden_states, _ = self.linear_1(hidden_states)
|
||||
hidden_states = self.act(hidden_states)
|
||||
|
||||
@@ -1,646 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from collections.abc import Iterable
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
from vllm.compilation.decorators import support_torch_compile
|
||||
from vllm.config import CacheConfig, VllmConfig
|
||||
from vllm.distributed import (
|
||||
get_pp_group,
|
||||
get_tensor_model_parallel_world_size,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.activation import SiluAndMul
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoE,
|
||||
fused_moe_make_expert_params_mapping,
|
||||
)
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.model_executor.layers.linear import (
|
||||
ColumnParallelLinear,
|
||||
MergedColumnParallelLinear,
|
||||
ReplicatedLinear,
|
||||
RowParallelLinear,
|
||||
)
|
||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||
from vllm.model_executor.layers.mamba.gdn.kimi_gdn_linear_attn import (
|
||||
KimiGatedDeltaNetAttention,
|
||||
)
|
||||
from vllm.model_executor.layers.mamba.mamba_utils import (
|
||||
MambaStateCopyFunc,
|
||||
MambaStateCopyFuncCalculator,
|
||||
MambaStateDtypeCalculator,
|
||||
MambaStateShapeCalculator,
|
||||
)
|
||||
from vllm.model_executor.layers.mla import MLAModules, MultiHeadLatentAttentionWrapper
|
||||
from vllm.model_executor.layers.quantization.base_config import QuantizationConfig
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
ParallelLMHead,
|
||||
VocabParallelEmbedding,
|
||||
)
|
||||
from vllm.model_executor.model_loader.weight_utils import (
|
||||
default_weight_loader,
|
||||
maybe_remap_kv_scale_name,
|
||||
)
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.transformers_utils.configs.kimi_linear import KimiLinearConfig
|
||||
|
||||
from .interfaces import HasInnerState, IsHybrid, MixtureOfExperts, SupportsPP
|
||||
from .utils import (
|
||||
AutoWeightsLoader,
|
||||
PPMissingLayer,
|
||||
get_spec_layer_idx_from_weight_name,
|
||||
is_pp_missing_parameter,
|
||||
make_layers,
|
||||
maybe_prefix,
|
||||
)
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class KimiMLP(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
hidden_size: int,
|
||||
intermediate_size: int,
|
||||
hidden_act: str,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
reduce_results: bool = True,
|
||||
prefix: str = "",
|
||||
) -> None:
|
||||
super().__init__()
|
||||
|
||||
self.gate_up_proj = MergedColumnParallelLinear(
|
||||
hidden_size,
|
||||
[intermediate_size] * 2,
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.gate_up_proj",
|
||||
)
|
||||
self.down_proj = RowParallelLinear(
|
||||
intermediate_size,
|
||||
hidden_size,
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
reduce_results=reduce_results,
|
||||
prefix=f"{prefix}.down_proj",
|
||||
)
|
||||
if hidden_act != "silu":
|
||||
raise ValueError(
|
||||
f"Unsupported activation: {hidden_act}. Only silu is supported for now."
|
||||
)
|
||||
self.act_fn = SiluAndMul()
|
||||
|
||||
def forward(self, x):
|
||||
gate_up, _ = self.gate_up_proj(x)
|
||||
x = self.act_fn(gate_up)
|
||||
x, _ = self.down_proj(x)
|
||||
return x
|
||||
|
||||
|
||||
class KimiMoE(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
config: KimiLinearConfig,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
prefix: str = "",
|
||||
layer_idx: int = 0,
|
||||
):
|
||||
super().__init__()
|
||||
hidden_size = config.hidden_size
|
||||
intermediate_size = config.intermediate_size
|
||||
moe_intermediate_size = config.moe_intermediate_size
|
||||
num_experts = config.num_experts
|
||||
moe_renormalize = config.moe_renormalize
|
||||
self.tp_size = get_tensor_model_parallel_world_size()
|
||||
self.routed_scaling_factor = config.routed_scaling_factor
|
||||
self.num_shared_experts = config.num_shared_experts
|
||||
self.layer_idx = layer_idx
|
||||
|
||||
if config.hidden_act != "silu":
|
||||
raise ValueError(
|
||||
f"Unsupported activation: {config.hidden_act}. "
|
||||
"Only silu is supported for now."
|
||||
)
|
||||
|
||||
# Gate always runs at half / full precision for now.
|
||||
self.gate = ReplicatedLinear(
|
||||
hidden_size,
|
||||
num_experts,
|
||||
bias=False,
|
||||
quant_config=None,
|
||||
prefix=f"{prefix}.gate",
|
||||
)
|
||||
|
||||
self.gate.e_score_correction_bias = nn.Parameter(torch.empty(num_experts))
|
||||
|
||||
if self.num_shared_experts is not None:
|
||||
intermediate_size = moe_intermediate_size * self.num_shared_experts
|
||||
self.shared_experts = KimiMLP(
|
||||
hidden_size=config.hidden_size,
|
||||
intermediate_size=intermediate_size,
|
||||
hidden_act=config.hidden_act,
|
||||
quant_config=quant_config,
|
||||
reduce_results=False,
|
||||
prefix=f"{prefix}.shared_experts",
|
||||
)
|
||||
else:
|
||||
self.shared_experts = None
|
||||
|
||||
self.experts = FusedMoE(
|
||||
shared_experts=self.shared_experts,
|
||||
num_experts=num_experts,
|
||||
top_k=config.num_experts_per_token,
|
||||
hidden_size=hidden_size,
|
||||
intermediate_size=moe_intermediate_size,
|
||||
renormalize=moe_renormalize,
|
||||
quant_config=quant_config,
|
||||
use_grouped_topk=config.use_grouped_topk,
|
||||
num_expert_group=config.num_expert_group,
|
||||
topk_group=config.topk_group,
|
||||
prefix=f"{prefix}.experts",
|
||||
scoring_func=config.moe_router_activation_func,
|
||||
e_score_correction_bias=self.gate.e_score_correction_bias,
|
||||
routed_scaling_factor=self.routed_scaling_factor,
|
||||
)
|
||||
|
||||
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
||||
num_tokens, hidden_size = hidden_states.shape
|
||||
hidden_states = hidden_states.view(-1, hidden_size)
|
||||
router_logits, _ = self.gate(hidden_states)
|
||||
final_hidden_states = self.experts(
|
||||
hidden_states=hidden_states, router_logits=router_logits
|
||||
)
|
||||
return final_hidden_states.view(num_tokens, hidden_size)
|
||||
|
||||
|
||||
class KimiMLAAttention(nn.Module):
|
||||
"""
|
||||
Main reference: DeepseekV2 vllm Implementation
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: KimiLinearConfig,
|
||||
hidden_size: int,
|
||||
num_heads: int,
|
||||
qk_nope_head_dim: int,
|
||||
qk_rope_head_dim: int,
|
||||
v_head_dim: int,
|
||||
q_lora_rank: int | None,
|
||||
kv_lora_rank: int,
|
||||
use_nope: bool = False,
|
||||
cache_config: CacheConfig | None = None,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
prefix: str = "",
|
||||
**kwargs,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.hidden_size = hidden_size
|
||||
self.qk_nope_head_dim = qk_nope_head_dim
|
||||
self.qk_rope_head_dim = qk_rope_head_dim
|
||||
self.qk_head_dim = qk_nope_head_dim + qk_rope_head_dim
|
||||
self.v_head_dim = v_head_dim
|
||||
self.q_lora_rank = q_lora_rank
|
||||
self.kv_lora_rank = kv_lora_rank
|
||||
self.num_heads = num_heads
|
||||
tp_size = get_tensor_model_parallel_world_size()
|
||||
self.num_local_heads = num_heads // tp_size
|
||||
self.scaling = self.qk_head_dim**-0.5
|
||||
self.use_nope = use_nope
|
||||
assert self.use_nope is True
|
||||
assert self.q_lora_rank is None
|
||||
assert num_heads % tp_size == 0
|
||||
self.kv_a_proj_with_mqa = ReplicatedLinear(
|
||||
self.hidden_size,
|
||||
self.kv_lora_rank + self.qk_rope_head_dim,
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.kv_a_proj_with_mqa",
|
||||
)
|
||||
self.q_proj = ColumnParallelLinear(
|
||||
self.hidden_size,
|
||||
self.num_heads * self.qk_head_dim,
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.q_proj",
|
||||
)
|
||||
self.kv_a_layernorm = RMSNorm(
|
||||
self.kv_lora_rank,
|
||||
eps=config.rms_norm_eps,
|
||||
)
|
||||
self.kv_b_proj = ColumnParallelLinear(
|
||||
self.kv_lora_rank,
|
||||
self.num_heads * (self.qk_nope_head_dim + self.v_head_dim),
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.kv_b_proj",
|
||||
)
|
||||
self.o_proj = RowParallelLinear(
|
||||
self.num_heads * self.v_head_dim,
|
||||
self.hidden_size,
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.o_proj",
|
||||
)
|
||||
|
||||
mla_modules = MLAModules(
|
||||
kv_a_layernorm=self.kv_a_layernorm,
|
||||
kv_b_proj=self.kv_b_proj,
|
||||
rotary_emb=None,
|
||||
o_proj=self.o_proj,
|
||||
fused_qkv_a_proj=None,
|
||||
kv_a_proj_with_mqa=self.kv_a_proj_with_mqa,
|
||||
q_a_layernorm=None,
|
||||
q_b_proj=None,
|
||||
q_proj=self.q_proj,
|
||||
indexer=None,
|
||||
is_sparse=False,
|
||||
topk_indices_buffer=None,
|
||||
)
|
||||
self.mla_attn = MultiHeadLatentAttentionWrapper(
|
||||
self.hidden_size,
|
||||
self.num_local_heads,
|
||||
self.scaling,
|
||||
self.qk_nope_head_dim,
|
||||
self.qk_rope_head_dim,
|
||||
self.v_head_dim,
|
||||
self.q_lora_rank,
|
||||
self.kv_lora_rank,
|
||||
mla_modules,
|
||||
cache_config,
|
||||
quant_config,
|
||||
prefix,
|
||||
)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
positions: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
output: torch.Tensor,
|
||||
) -> None:
|
||||
output[:] = self.mla_attn(positions, hidden_states)
|
||||
|
||||
|
||||
class KimiDecoderLayer(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
config: KimiLinearConfig,
|
||||
vllm_config: VllmConfig,
|
||||
prefix: str = "",
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.hidden_size = config.hidden_size
|
||||
|
||||
self.is_moe = config.is_moe
|
||||
layer_idx = int(prefix.rsplit(".", 1)[1])
|
||||
model_config = vllm_config.model_config
|
||||
cache_config = vllm_config.cache_config
|
||||
quant_config = vllm_config.quant_config
|
||||
|
||||
if config.is_kda_layer(layer_idx):
|
||||
self.self_attn = KimiGatedDeltaNetAttention(
|
||||
config,
|
||||
vllm_config,
|
||||
prefix=f"{prefix}.self_attn",
|
||||
)
|
||||
else:
|
||||
self.self_attn = KimiMLAAttention(
|
||||
layer_idx=layer_idx,
|
||||
hidden_size=self.hidden_size,
|
||||
num_heads=config.num_attention_heads,
|
||||
quant_config=quant_config,
|
||||
cache_config=cache_config,
|
||||
model_config=model_config,
|
||||
prefix=f"{prefix}.self_attn",
|
||||
config=config,
|
||||
qk_nope_head_dim=config.qk_nope_head_dim,
|
||||
qk_rope_head_dim=config.qk_rope_head_dim,
|
||||
v_head_dim=config.v_head_dim,
|
||||
q_lora_rank=config.q_lora_rank,
|
||||
kv_lora_rank=config.kv_lora_rank,
|
||||
use_nope=config.mla_use_nope,
|
||||
)
|
||||
|
||||
if (
|
||||
self.is_moe
|
||||
and config.num_experts is not None
|
||||
and layer_idx >= config.first_k_dense_replace
|
||||
and layer_idx % config.moe_layer_freq == 0
|
||||
):
|
||||
self.block_sparse_moe = KimiMoE(
|
||||
config=config,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.block_sparse_moe",
|
||||
)
|
||||
self.mlp = self.block_sparse_moe
|
||||
else:
|
||||
self.mlp = KimiMLP(
|
||||
hidden_size=self.hidden_size,
|
||||
intermediate_size=config.intermediate_size,
|
||||
hidden_act=config.hidden_act,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.mlp",
|
||||
)
|
||||
self.input_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
self.post_attention_layernorm = RMSNorm(
|
||||
config.hidden_size, eps=config.rms_norm_eps
|
||||
)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
positions: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
residual: torch.Tensor | None,
|
||||
**kwargs,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
# Self Attention
|
||||
if residual is None:
|
||||
residual = hidden_states
|
||||
hidden_states = self.input_layernorm(hidden_states)
|
||||
else:
|
||||
hidden_states, residual = self.input_layernorm(hidden_states, residual)
|
||||
|
||||
attn_output = torch.empty_like(hidden_states)
|
||||
self.self_attn(
|
||||
hidden_states=hidden_states,
|
||||
positions=positions,
|
||||
output=attn_output,
|
||||
)
|
||||
hidden_states = attn_output
|
||||
|
||||
# Fully Connected
|
||||
hidden_states, residual = self.post_attention_layernorm(hidden_states, residual)
|
||||
hidden_states = self.mlp(hidden_states)
|
||||
return hidden_states, residual
|
||||
|
||||
|
||||
@support_torch_compile
|
||||
class KimiLinearModel(nn.Module):
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
super().__init__()
|
||||
|
||||
config = vllm_config.model_config.hf_text_config
|
||||
self.config = config
|
||||
|
||||
self.vocab_size = config.vocab_size
|
||||
|
||||
if get_pp_group().is_first_rank:
|
||||
self.embed_tokens = VocabParallelEmbedding(
|
||||
config.vocab_size,
|
||||
config.hidden_size,
|
||||
prefix=f"{prefix}.embed_tokens",
|
||||
)
|
||||
else:
|
||||
self.embed_tokens = PPMissingLayer()
|
||||
|
||||
def get_layer(prefix: str):
|
||||
return KimiDecoderLayer(
|
||||
config,
|
||||
vllm_config,
|
||||
prefix,
|
||||
)
|
||||
|
||||
self.start_layer, self.end_layer, self.layers = make_layers(
|
||||
config.num_hidden_layers,
|
||||
get_layer,
|
||||
prefix=f"{prefix}.layers",
|
||||
)
|
||||
|
||||
if get_pp_group().is_last_rank:
|
||||
self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
else:
|
||||
self.norm = PPMissingLayer()
|
||||
|
||||
world_size = get_tensor_model_parallel_world_size()
|
||||
assert config.num_attention_heads % world_size == 0, (
|
||||
"num_attention_heads must be divisible by world_size"
|
||||
)
|
||||
|
||||
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
|
||||
return self.embed_tokens(input_ids)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.Tensor | None,
|
||||
positions: torch.Tensor,
|
||||
intermediate_tensors: IntermediateTensors | None,
|
||||
inputs_embeds: torch.Tensor | None = None,
|
||||
**kwargs,
|
||||
) -> torch.Tensor:
|
||||
if get_pp_group().is_first_rank:
|
||||
if inputs_embeds is not None:
|
||||
hidden_states = inputs_embeds
|
||||
else:
|
||||
hidden_states = self.embed_input_ids(input_ids)
|
||||
residual = None
|
||||
else:
|
||||
assert intermediate_tensors is not None
|
||||
hidden_states = intermediate_tensors["hidden_states"]
|
||||
residual = intermediate_tensors["residual"]
|
||||
|
||||
for _, layer in enumerate(self.layers[self.start_layer : self.end_layer]):
|
||||
hidden_states, residual = layer(
|
||||
positions=positions,
|
||||
hidden_states=hidden_states,
|
||||
residual=residual,
|
||||
)
|
||||
|
||||
if not get_pp_group().is_last_rank:
|
||||
return IntermediateTensors(
|
||||
{"hidden_states": hidden_states, "residual": residual}
|
||||
)
|
||||
|
||||
hidden_states, _ = self.norm(hidden_states, residual)
|
||||
return hidden_states
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
stacked_params_mapping = [
|
||||
# (param_name, shard_name, shard_id)
|
||||
(".gate_up_proj", ".gate_proj", 0),
|
||||
(".gate_up_proj", ".up_proj", 1),
|
||||
]
|
||||
if self.config.is_moe:
|
||||
# Params for weights, fp8 weight scales, fp8 activation scales
|
||||
# (param_name, weight_name, expert_id, shard_id)
|
||||
expert_params_mapping = fused_moe_make_expert_params_mapping(
|
||||
self,
|
||||
ckpt_gate_proj_name="w1",
|
||||
ckpt_down_proj_name="w2",
|
||||
ckpt_up_proj_name="w3",
|
||||
num_experts=self.config.num_experts,
|
||||
)
|
||||
else:
|
||||
expert_params_mapping = []
|
||||
params_dict = dict(self.named_parameters())
|
||||
loaded_params: set[str] = set()
|
||||
for args in weights:
|
||||
name, loaded_weight = args[:2]
|
||||
kwargs = args[2] if len(args) > 2 else {}
|
||||
if "rotary_emb.inv_freq" in name:
|
||||
continue
|
||||
|
||||
spec_layer = get_spec_layer_idx_from_weight_name(self.config, name)
|
||||
if spec_layer is not None:
|
||||
continue # skip spec decode layers for main model
|
||||
if "rotary_emb.cos_cached" in name or "rotary_emb.sin_cached" in name:
|
||||
# Models trained using ColossalAI may include these tensors in
|
||||
# the checkpoint. Skip them.
|
||||
continue
|
||||
for param_name, weight_name, shard_id in stacked_params_mapping:
|
||||
if weight_name not in name:
|
||||
continue
|
||||
# We have mlp.experts[0].gate_proj in the checkpoint.
|
||||
# Since we handle the experts below in expert_params_mapping,
|
||||
# we need to skip here BEFORE we update the name, otherwise
|
||||
# name will be updated to mlp.experts[0].gate_up_proj, which
|
||||
# will then be updated below in expert_params_mapping
|
||||
# for mlp.experts[0].gate_gate_up_proj, which breaks load.
|
||||
if ("mlp.experts." in name) and name not in params_dict:
|
||||
continue
|
||||
name = name.replace(weight_name, param_name)
|
||||
# Skip loading extra bias for GPTQ models.
|
||||
if name.endswith(".bias") and name not in params_dict:
|
||||
continue
|
||||
if is_pp_missing_parameter(name, self):
|
||||
continue
|
||||
param = params_dict[name]
|
||||
weight_loader = param.weight_loader
|
||||
weight_loader(param, loaded_weight, shard_id)
|
||||
break
|
||||
else:
|
||||
for idx, (param_name, weight_name, expert_id, shard_id) in enumerate(
|
||||
expert_params_mapping
|
||||
):
|
||||
if weight_name not in name:
|
||||
continue
|
||||
name = name.replace(weight_name, param_name)
|
||||
if is_pp_missing_parameter(name, self):
|
||||
continue
|
||||
param = params_dict[name]
|
||||
weight_loader = param.weight_loader
|
||||
weight_loader(
|
||||
param,
|
||||
loaded_weight,
|
||||
name,
|
||||
expert_id=expert_id,
|
||||
shard_id=shard_id,
|
||||
)
|
||||
break
|
||||
else:
|
||||
# Skip loading extra bias for GPTQ models.
|
||||
if (
|
||||
name.endswith(".bias")
|
||||
and name not in params_dict
|
||||
and not self.config.is_linear_attn
|
||||
): # noqa: E501
|
||||
continue
|
||||
# Remapping the name of FP8 kv-scale.
|
||||
name = maybe_remap_kv_scale_name(name, params_dict)
|
||||
if name is None:
|
||||
continue
|
||||
if is_pp_missing_parameter(name, self):
|
||||
continue
|
||||
|
||||
param = params_dict[name]
|
||||
weight_loader = getattr(
|
||||
param, "weight_loader", default_weight_loader
|
||||
)
|
||||
weight_loader(param, loaded_weight, **kwargs)
|
||||
loaded_params.add(name)
|
||||
return loaded_params
|
||||
|
||||
|
||||
class KimiLinearForCausalLM(
|
||||
nn.Module, HasInnerState, SupportsPP, MixtureOfExperts, IsHybrid
|
||||
):
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
super().__init__()
|
||||
self.model_config = vllm_config.model_config
|
||||
self.vllm_config = vllm_config
|
||||
self.config = self.model_config.hf_config
|
||||
quant_config = vllm_config.quant_config
|
||||
self.quant_config = quant_config
|
||||
self.model = KimiLinearModel(
|
||||
vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model")
|
||||
)
|
||||
if get_pp_group().is_last_rank:
|
||||
self.lm_head = ParallelLMHead(
|
||||
self.config.vocab_size,
|
||||
self.config.hidden_size,
|
||||
quant_config=quant_config,
|
||||
prefix=maybe_prefix(prefix, "lm_head"),
|
||||
)
|
||||
else:
|
||||
self.lm_head = PPMissingLayer()
|
||||
logit_scale = getattr(self.config, "logit_scale", 1.0)
|
||||
self.logits_processor = LogitsProcessor(
|
||||
self.config.vocab_size, scale=logit_scale
|
||||
)
|
||||
|
||||
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
|
||||
return self.model.embed_input_ids(input_ids)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.Tensor | None,
|
||||
positions: torch.Tensor,
|
||||
intermediate_tensors: IntermediateTensors | None = None,
|
||||
inputs_embeds: torch.Tensor | None = None,
|
||||
**kwargs,
|
||||
) -> torch.Tensor | IntermediateTensors:
|
||||
hidden_states = self.model(
|
||||
input_ids, positions, intermediate_tensors, inputs_embeds, **kwargs
|
||||
)
|
||||
return hidden_states
|
||||
|
||||
@classmethod
|
||||
def get_mamba_state_dtype_from_config(
|
||||
cls,
|
||||
vllm_config: "VllmConfig",
|
||||
) -> tuple[torch.dtype, torch.dtype]:
|
||||
return MambaStateDtypeCalculator.kda_state_dtype(
|
||||
vllm_config.model_config.dtype, vllm_config.cache_config.mamba_cache_dtype
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def get_mamba_state_shape_from_config(
|
||||
cls, vllm_config: "VllmConfig"
|
||||
) -> tuple[tuple[int, ...], tuple[int, ...]]:
|
||||
parallel_config = vllm_config.parallel_config
|
||||
hf_config = vllm_config.model_config.hf_config
|
||||
tp_size = parallel_config.tensor_parallel_size
|
||||
num_spec = (
|
||||
vllm_config.speculative_config.num_speculative_tokens
|
||||
if vllm_config.speculative_config
|
||||
else 0
|
||||
)
|
||||
return MambaStateShapeCalculator.kda_state_shape(
|
||||
tp_size,
|
||||
hf_config.linear_attn_config["num_heads"],
|
||||
hf_config.linear_attn_config["head_dim"],
|
||||
conv_kernel_size=hf_config.linear_attn_config["short_conv_kernel_size"],
|
||||
num_spec=num_spec,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def get_mamba_state_copy_func(
|
||||
cls,
|
||||
) -> tuple[MambaStateCopyFunc, MambaStateCopyFunc]:
|
||||
return MambaStateCopyFuncCalculator.kda_state_copy_func()
|
||||
|
||||
def compute_logits(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
) -> torch.Tensor | None:
|
||||
return self.logits_processor(self.lm_head, hidden_states)
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
loader = AutoWeightsLoader(
|
||||
self,
|
||||
skip_prefixes=(["lm_head."] if self.config.tie_word_embeddings else None),
|
||||
)
|
||||
return loader.load_weights(weights)
|
||||
@@ -137,7 +137,10 @@ _TEXT_GENERATION_MODELS = {
|
||||
"IQuestLoopCoderForCausalLM": ("iquest_loopcoder", "IQuestLoopCoderForCausalLM"),
|
||||
"Jais2ForCausalLM": ("jais2", "Jais2ForCausalLM"),
|
||||
"JambaForCausalLM": ("jamba", "JambaForCausalLM"),
|
||||
"KimiLinearForCausalLM": ("kimi_linear", "KimiLinearForCausalLM"),
|
||||
"KimiLinearForCausalLM": (
|
||||
"vllm.models.kimi_k3",
|
||||
"KimiLinearForCausalLM",
|
||||
),
|
||||
"Lfm2ForCausalLM": ("lfm2", "Lfm2ForCausalLM"),
|
||||
"Lfm2MoeForCausalLM": ("lfm2_moe", "Lfm2MoeForCausalLM"),
|
||||
"LagunaForCausalLM": ("laguna", "LagunaForCausalLM"),
|
||||
@@ -460,6 +463,10 @@ _MULTIMODAL_MODELS = {
|
||||
),
|
||||
"KimiVLForConditionalGeneration": ("kimi_vl", "KimiVLForConditionalGeneration"),
|
||||
"KimiK25ForConditionalGeneration": ("kimi_k25", "KimiK25ForConditionalGeneration"),
|
||||
"KimiK3ForConditionalGeneration": (
|
||||
"vllm.models.kimi_k3",
|
||||
"KimiK3ForConditionalGeneration",
|
||||
),
|
||||
"MoonshotKimiaForCausalLM": ("kimi_audio", "KimiAudioForConditionalGeneration"),
|
||||
"MossTranscribeDiarizeForConditionalGeneration": (
|
||||
"moss_transcribe_diarize",
|
||||
@@ -609,6 +616,10 @@ _SPECULATIVE_DECODING_MODELS = {
|
||||
"DFlashDraftModel": ("qwen3_dflash", "DFlashQwen3ForCausalLM"),
|
||||
"DSparkDraftModel": ("vllm.models.deepseek_v4", "DSparkDeepseekV4ForCausalLM"),
|
||||
"Qwen3DSparkModel": ("qwen3_dspark", "Qwen3DSparkForCausalLM"),
|
||||
"K3DSparkModel": (
|
||||
"vllm.models.kimi_k3.nvidia.dspark_mla",
|
||||
"K3DSparkForCausalLM",
|
||||
),
|
||||
"DFlashLagunaForCausalLM": ("laguna_dflash", "DFlashLagunaForCausalLM"),
|
||||
"Gemma4DSparkModel": ("gemma4_dspark", "Gemma4DSparkForCausalLM"),
|
||||
"PEagleDraftModel": ("llama_eagle3", "Eagle3LlamaForCausalLM"),
|
||||
@@ -649,6 +660,7 @@ _SPECULATIVE_DECODING_MODELS = {
|
||||
"Qwen3_5MTP": ("qwen3_5_mtp", "Qwen3_5MTP"),
|
||||
"Qwen3_5MoeMTP": ("qwen3_5_mtp", "Qwen3_5MoeMTP"),
|
||||
"HYV3MTPModel": ("hy_v3_mtp", "HYV3MTP"),
|
||||
"KimiK3MTPModel": ("vllm.models.kimi_k3", "KimiK3MTP"),
|
||||
# Temporarily disabled.
|
||||
# # TODO(woosuk): Re-enable this once the MLP Speculator is supported in V1.
|
||||
# "MLPSpeculatorPreTrainedModel": ("mlp_speculator", "MLPSpeculator"),
|
||||
|
||||
Reference in New Issue
Block a user