[New model] Kimi K3 (#50000)
Signed-off-by: zjy0516 <riverclouds.zhu@qq.com> Signed-off-by: Isotr0py <Isotr0py@outlook.com> Signed-off-by: Bugen Zhao <i@bugenzhao.com> Signed-off-by: Jee Jee Li <jeejeelee@inferact.ai> Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai> Signed-off-by: Yongye Zhu <zyy1102000@gmail.com> Signed-off-by: wzhao18 <wzhao18.sz@gmail.com> Co-authored-by: Isotr0py <Isotr0py@outlook.com> Co-authored-by: Thien Tran <gau.nernst@yahoo.com.sg> Co-authored-by: Bugen Zhao <i@bugenzhao.com> Co-authored-by: Yongye Zhu <zyy1102000@gmail.com> Co-authored-by: Ziming Huang <zelda.huanghuang@gmail.com> Co-authored-by: Roger Wang <hey@rogerw.io> Co-authored-by: Isotr0py <mozf@inferact.ai> Co-authored-by: aoshen02 <aoshen02@users.noreply.github.com> Co-authored-by: Woosuk Kwon <woosuk@inferact.ai> Co-authored-by: Jee Jee Li <jeejeelee@inferact.ai> Co-authored-by: aoshen02 <aoshen@inferact.ai> Co-authored-by: Summer Yang <girasoleyang@gmail.com> Co-authored-by: Kevin H. Luu <khluu000@gmail.com> Co-authored-by: Bowen Wang <abmfy@icloud.com> Co-authored-by: gnovack <novackgm@gmail.com> Co-authored-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai> Co-authored-by: xiaozhoupy <peiyuanzhou1994@gmail.com> Co-authored-by: Roy Wang <yasong.wang@inferact.ai> Co-authored-by: Jeff (Junze) Ma <93145857+majunze2001@users.noreply.github.com> Co-authored-by: Codex <codex@openai.com> Co-authored-by: Wei Zhao <51183510+wzhao18@users.noreply.github.com>
This commit is contained in:
co-authored by
Isotr0py
Thien Tran
Bugen Zhao
Yongye Zhu
Ziming Huang
Roger Wang
Isotr0py
aoshen02
Woosuk Kwon
Jee Jee Li
aoshen02
Summer Yang
Kevin H. Luu
Bowen Wang
gnovack
Nick Hill
Yifan Qiao
xiaozhoupy
Roy Wang
Jeff Ma
Codex
Wei Zhao
parent
0c64be8873
commit
aeeb36b1f1
@@ -1026,6 +1026,10 @@ _MULTIMODAL_EXAMPLE_MODELS = {
|
||||
"moonshotai/Kimi-K2.5",
|
||||
trust_remote_code=True,
|
||||
),
|
||||
"KimiK3ForConditionalGeneration": _HfExamplesInfo(
|
||||
"moonshotai/Kimi-K3",
|
||||
trust_remote_code=True,
|
||||
),
|
||||
"KimiVLForConditionalGeneration": _HfExamplesInfo(
|
||||
"moonshotai/Kimi-VL-A3B-Instruct",
|
||||
extras={"thinking": "moonshotai/Kimi-VL-A3B-Thinking"},
|
||||
@@ -1466,6 +1470,14 @@ _SPECULATIVE_DECODING_EXAMPLE_MODELS = {
|
||||
is_available_online=False,
|
||||
use_original_num_layers=True, # DSpark has >1 draft block
|
||||
),
|
||||
"K3DSparkModel": _HfExamplesInfo(
|
||||
"moonshotai/Kimi-K3",
|
||||
speculative_model="Inferact/Kimi-K3-DSpark",
|
||||
use_original_num_layers=True, # DSpark has >1 draft block
|
||||
trust_remote_code=True,
|
||||
# FIXME: Investigate the NVML failure in CI.
|
||||
is_available_online=False,
|
||||
),
|
||||
"Qwen3DSparkModel": _HfExamplesInfo(
|
||||
"Qwen/Qwen3-8B",
|
||||
speculative_model="deepseek-ai/dspark_qwen3_8b_block7",
|
||||
@@ -1664,6 +1676,12 @@ _SPECULATIVE_DECODING_EXAMPLE_MODELS = {
|
||||
trust_remote_code=True,
|
||||
max_model_len=4096,
|
||||
),
|
||||
"KimiK3MTPModel": _HfExamplesInfo(
|
||||
"moonshotai/Kimi-K3",
|
||||
speculative_model="moonshotai/Kimi-K3",
|
||||
trust_remote_code=True,
|
||||
is_available_online=False,
|
||||
),
|
||||
"LongCatFlashMTPModel": _HfExamplesInfo(
|
||||
"meituan-longcat/LongCat-Flash-Chat",
|
||||
trust_remote_code=True,
|
||||
|
||||
@@ -0,0 +1,143 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
from vllm.compilation.wrapper import TorchCompileWithNoGuardsWrapper
|
||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||
from vllm.model_executor.models.qwen3_dspark import DSparkMarkovHead
|
||||
from vllm.model_executor.models.registry import ModelRegistry
|
||||
from vllm.models.kimi_k3.nvidia import dspark_mla
|
||||
from vllm.models.kimi_k3.nvidia.dspark_mla import K3DSparkForCausalLM, K3DSparkModel
|
||||
|
||||
|
||||
def test_dspark_mla_uses_compile_free_model_entrypoint():
|
||||
assert ModelRegistry._try_load_model_cls("K3DSparkModel") is K3DSparkForCausalLM
|
||||
assert not issubclass(K3DSparkModel, TorchCompileWithNoGuardsWrapper)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("checkpoint_name", "runtime_name", "shard_id"),
|
||||
[
|
||||
(
|
||||
"layers.0.self_attn.q_a_proj.weight",
|
||||
"model.layers.0.self_attn.fused_qkv_a_proj.weight",
|
||||
0,
|
||||
),
|
||||
(
|
||||
"layers.0.self_attn.kv_a_proj_with_mqa.weight",
|
||||
"model.layers.0.self_attn.fused_qkv_a_proj.weight",
|
||||
1,
|
||||
),
|
||||
(
|
||||
"layers.0.mlp.gate_proj.weight",
|
||||
"model.layers.0.mlp.gate_up_proj.weight",
|
||||
0,
|
||||
),
|
||||
(
|
||||
"layers.0.mlp.up_proj.weight",
|
||||
"model.layers.0.mlp.gate_up_proj.weight",
|
||||
1,
|
||||
),
|
||||
("context_proj.weight", "model.context_proj.weight", None),
|
||||
],
|
||||
)
|
||||
def test_dspark_mla_checkpoint_weight_mapping(checkpoint_name, runtime_name, shard_id):
|
||||
assert K3DSparkForCausalLM.hf_to_vllm_mapper._map_name_with_shard(
|
||||
checkpoint_name
|
||||
) == (runtime_name, shard_id)
|
||||
|
||||
|
||||
def test_dspark_mla_shares_frozen_target_weights_and_skips_training_head():
|
||||
assert not K3DSparkForCausalLM.has_own_embed_tokens
|
||||
assert not K3DSparkForCausalLM.has_own_lm_head
|
||||
assert set(K3DSparkForCausalLM.checkpoint_skip_substrs) == {
|
||||
"confidence_head",
|
||||
"embed_tokens",
|
||||
"lm_head",
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
def test_dspark_markov_head_is_replicated(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
from vllm.model_executor.layers import logits_processor, vocab_parallel_embedding
|
||||
|
||||
monkeypatch.setattr(
|
||||
vocab_parallel_embedding, "get_tensor_model_parallel_rank", lambda: 3
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
vocab_parallel_embedding,
|
||||
"get_tensor_model_parallel_world_size",
|
||||
lambda: 8,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
logits_processor,
|
||||
"get_current_vllm_config",
|
||||
lambda: SimpleNamespace(model_config=None),
|
||||
)
|
||||
|
||||
head = DSparkMarkovHead(128, 128, 8, prefix="markov_head")
|
||||
assert head.markov_w2.tp_size == 1
|
||||
assert head.markov_w1.weight.shape == (128, 8)
|
||||
assert head.markov_w2.weight.shape == (128, 8)
|
||||
|
||||
def fail_collective(*args, **kwargs):
|
||||
raise AssertionError("replicated Markov head must not invoke TP collectives")
|
||||
|
||||
monkeypatch.setattr(
|
||||
vocab_parallel_embedding,
|
||||
"tensor_model_parallel_all_reduce",
|
||||
fail_collective,
|
||||
)
|
||||
logits_processor = LogitsProcessor(128)
|
||||
monkeypatch.setattr(logits_processor, "_gather_logits", fail_collective)
|
||||
|
||||
markov_embed = head.embed(torch.tensor([1, 2]))
|
||||
bias = head.bias(markov_embed, logits_processor)
|
||||
assert markov_embed.shape == (2, 8)
|
||||
assert bias.shape == (2, 128)
|
||||
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
def test_k3_dspark_uses_replicated_markov_head(monkeypatch: pytest.MonkeyPatch):
|
||||
markov_head_calls = []
|
||||
|
||||
class DummyModule(nn.Module):
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__()
|
||||
|
||||
def make_markov_head(*args, **kwargs):
|
||||
markov_head_calls.append((args, kwargs))
|
||||
return DummyModule()
|
||||
|
||||
monkeypatch.setattr(dspark_mla, "get_draft_quant_config", lambda _: None)
|
||||
monkeypatch.setattr(dspark_mla, "ReplicatedLinear", DummyModule)
|
||||
monkeypatch.setattr(dspark_mla, "RMSNorm", DummyModule)
|
||||
monkeypatch.setattr(dspark_mla, "K3DSparkDecoderLayer", DummyModule)
|
||||
monkeypatch.setattr(dspark_mla, "DSparkMarkovHead", make_markov_head)
|
||||
|
||||
config = SimpleNamespace(
|
||||
target_hidden_size=16,
|
||||
num_target_layers=2,
|
||||
hidden_size=8,
|
||||
rms_norm_eps=1e-6,
|
||||
num_hidden_layers=1,
|
||||
vocab_size=128,
|
||||
draft_vocab_size=128,
|
||||
markov_rank=4,
|
||||
)
|
||||
vllm_config = SimpleNamespace(
|
||||
speculative_config=SimpleNamespace(
|
||||
draft_model_config=SimpleNamespace(hf_config=config)
|
||||
)
|
||||
)
|
||||
|
||||
K3DSparkModel(vllm_config=vllm_config, start_layer_id=0, prefix="model")
|
||||
|
||||
assert len(markov_head_calls) == 1
|
||||
Reference in New Issue
Block a user