[New model] Kimi K3 (#50000)

Signed-off-by: zjy0516 <riverclouds.zhu@qq.com>
Signed-off-by: Isotr0py <Isotr0py@outlook.com>
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
Signed-off-by: Jee Jee Li <jeejeelee@inferact.ai>
Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai>
Signed-off-by: Yongye Zhu <zyy1102000@gmail.com>
Signed-off-by: wzhao18 <wzhao18.sz@gmail.com>
Co-authored-by: Isotr0py <Isotr0py@outlook.com>
Co-authored-by: Thien Tran <gau.nernst@yahoo.com.sg>
Co-authored-by: Bugen Zhao <i@bugenzhao.com>
Co-authored-by: Yongye Zhu <zyy1102000@gmail.com>
Co-authored-by: Ziming Huang <zelda.huanghuang@gmail.com>
Co-authored-by: Roger Wang <hey@rogerw.io>
Co-authored-by: Isotr0py <mozf@inferact.ai>
Co-authored-by: aoshen02 <aoshen02@users.noreply.github.com>
Co-authored-by: Woosuk Kwon <woosuk@inferact.ai>
Co-authored-by: Jee Jee Li <jeejeelee@inferact.ai>
Co-authored-by: aoshen02 <aoshen@inferact.ai>
Co-authored-by: Summer Yang <girasoleyang@gmail.com>
Co-authored-by: Kevin H. Luu <khluu000@gmail.com>
Co-authored-by: Bowen Wang <abmfy@icloud.com>
Co-authored-by: gnovack <novackgm@gmail.com>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai>
Co-authored-by: xiaozhoupy <peiyuanzhou1994@gmail.com>
Co-authored-by: Roy Wang <yasong.wang@inferact.ai>
Co-authored-by: Jeff (Junze) Ma <93145857+majunze2001@users.noreply.github.com>
Co-authored-by: Codex <codex@openai.com>
Co-authored-by: Wei Zhao <51183510+wzhao18@users.noreply.github.com>
This commit is contained in:
Jiangyun Zhu
2026-07-30 03:49:58 -07:00
committed by GitHub
co-authored by Isotr0py Thien Tran Bugen Zhao Yongye Zhu Ziming Huang Roger Wang Isotr0py aoshen02 Woosuk Kwon Jee Jee Li aoshen02 Summer Yang Kevin H. Luu Bowen Wang gnovack Nick Hill Yifan Qiao xiaozhoupy Roy Wang Jeff Ma Codex Wei Zhao
parent 0c64be8873
commit aeeb36b1f1
82 changed files with 2927 additions and 1345 deletions
+18
View File
@@ -1026,6 +1026,10 @@ _MULTIMODAL_EXAMPLE_MODELS = {
"moonshotai/Kimi-K2.5",
trust_remote_code=True,
),
"KimiK3ForConditionalGeneration": _HfExamplesInfo(
"moonshotai/Kimi-K3",
trust_remote_code=True,
),
"KimiVLForConditionalGeneration": _HfExamplesInfo(
"moonshotai/Kimi-VL-A3B-Instruct",
extras={"thinking": "moonshotai/Kimi-VL-A3B-Thinking"},
@@ -1466,6 +1470,14 @@ _SPECULATIVE_DECODING_EXAMPLE_MODELS = {
is_available_online=False,
use_original_num_layers=True, # DSpark has >1 draft block
),
"K3DSparkModel": _HfExamplesInfo(
"moonshotai/Kimi-K3",
speculative_model="Inferact/Kimi-K3-DSpark",
use_original_num_layers=True, # DSpark has >1 draft block
trust_remote_code=True,
# FIXME: Investigate the NVML failure in CI.
is_available_online=False,
),
"Qwen3DSparkModel": _HfExamplesInfo(
"Qwen/Qwen3-8B",
speculative_model="deepseek-ai/dspark_qwen3_8b_block7",
@@ -1664,6 +1676,12 @@ _SPECULATIVE_DECODING_EXAMPLE_MODELS = {
trust_remote_code=True,
max_model_len=4096,
),
"KimiK3MTPModel": _HfExamplesInfo(
"moonshotai/Kimi-K3",
speculative_model="moonshotai/Kimi-K3",
trust_remote_code=True,
is_available_online=False,
),
"LongCatFlashMTPModel": _HfExamplesInfo(
"meituan-longcat/LongCat-Flash-Chat",
trust_remote_code=True,
+143
View File
@@ -0,0 +1,143 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from types import SimpleNamespace
import pytest
import torch
import torch.nn as nn
from vllm.compilation.wrapper import TorchCompileWithNoGuardsWrapper
from vllm.model_executor.layers.logits_processor import LogitsProcessor
from vllm.model_executor.models.qwen3_dspark import DSparkMarkovHead
from vllm.model_executor.models.registry import ModelRegistry
from vllm.models.kimi_k3.nvidia import dspark_mla
from vllm.models.kimi_k3.nvidia.dspark_mla import K3DSparkForCausalLM, K3DSparkModel
def test_dspark_mla_uses_compile_free_model_entrypoint():
assert ModelRegistry._try_load_model_cls("K3DSparkModel") is K3DSparkForCausalLM
assert not issubclass(K3DSparkModel, TorchCompileWithNoGuardsWrapper)
@pytest.mark.parametrize(
("checkpoint_name", "runtime_name", "shard_id"),
[
(
"layers.0.self_attn.q_a_proj.weight",
"model.layers.0.self_attn.fused_qkv_a_proj.weight",
0,
),
(
"layers.0.self_attn.kv_a_proj_with_mqa.weight",
"model.layers.0.self_attn.fused_qkv_a_proj.weight",
1,
),
(
"layers.0.mlp.gate_proj.weight",
"model.layers.0.mlp.gate_up_proj.weight",
0,
),
(
"layers.0.mlp.up_proj.weight",
"model.layers.0.mlp.gate_up_proj.weight",
1,
),
("context_proj.weight", "model.context_proj.weight", None),
],
)
def test_dspark_mla_checkpoint_weight_mapping(checkpoint_name, runtime_name, shard_id):
assert K3DSparkForCausalLM.hf_to_vllm_mapper._map_name_with_shard(
checkpoint_name
) == (runtime_name, shard_id)
def test_dspark_mla_shares_frozen_target_weights_and_skips_training_head():
assert not K3DSparkForCausalLM.has_own_embed_tokens
assert not K3DSparkForCausalLM.has_own_lm_head
assert set(K3DSparkForCausalLM.checkpoint_skip_substrs) == {
"confidence_head",
"embed_tokens",
"lm_head",
}
@pytest.mark.cpu_test
def test_dspark_markov_head_is_replicated(
monkeypatch: pytest.MonkeyPatch,
):
from vllm.model_executor.layers import logits_processor, vocab_parallel_embedding
monkeypatch.setattr(
vocab_parallel_embedding, "get_tensor_model_parallel_rank", lambda: 3
)
monkeypatch.setattr(
vocab_parallel_embedding,
"get_tensor_model_parallel_world_size",
lambda: 8,
)
monkeypatch.setattr(
logits_processor,
"get_current_vllm_config",
lambda: SimpleNamespace(model_config=None),
)
head = DSparkMarkovHead(128, 128, 8, prefix="markov_head")
assert head.markov_w2.tp_size == 1
assert head.markov_w1.weight.shape == (128, 8)
assert head.markov_w2.weight.shape == (128, 8)
def fail_collective(*args, **kwargs):
raise AssertionError("replicated Markov head must not invoke TP collectives")
monkeypatch.setattr(
vocab_parallel_embedding,
"tensor_model_parallel_all_reduce",
fail_collective,
)
logits_processor = LogitsProcessor(128)
monkeypatch.setattr(logits_processor, "_gather_logits", fail_collective)
markov_embed = head.embed(torch.tensor([1, 2]))
bias = head.bias(markov_embed, logits_processor)
assert markov_embed.shape == (2, 8)
assert bias.shape == (2, 128)
@pytest.mark.cpu_test
def test_k3_dspark_uses_replicated_markov_head(monkeypatch: pytest.MonkeyPatch):
markov_head_calls = []
class DummyModule(nn.Module):
def __init__(self, *args, **kwargs):
super().__init__()
def make_markov_head(*args, **kwargs):
markov_head_calls.append((args, kwargs))
return DummyModule()
monkeypatch.setattr(dspark_mla, "get_draft_quant_config", lambda _: None)
monkeypatch.setattr(dspark_mla, "ReplicatedLinear", DummyModule)
monkeypatch.setattr(dspark_mla, "RMSNorm", DummyModule)
monkeypatch.setattr(dspark_mla, "K3DSparkDecoderLayer", DummyModule)
monkeypatch.setattr(dspark_mla, "DSparkMarkovHead", make_markov_head)
config = SimpleNamespace(
target_hidden_size=16,
num_target_layers=2,
hidden_size=8,
rms_norm_eps=1e-6,
num_hidden_layers=1,
vocab_size=128,
draft_vocab_size=128,
markov_rank=4,
)
vllm_config = SimpleNamespace(
speculative_config=SimpleNamespace(
draft_model_config=SimpleNamespace(hf_config=config)
)
)
K3DSparkModel(vllm_config=vllm_config, start_layer_id=0, prefix="model")
assert len(markov_head_calls) == 1