[New model] Kimi K3 (#50000)
Signed-off-by: zjy0516 <riverclouds.zhu@qq.com> Signed-off-by: Isotr0py <Isotr0py@outlook.com> Signed-off-by: Bugen Zhao <i@bugenzhao.com> Signed-off-by: Jee Jee Li <jeejeelee@inferact.ai> Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai> Signed-off-by: Yongye Zhu <zyy1102000@gmail.com> Signed-off-by: wzhao18 <wzhao18.sz@gmail.com> Co-authored-by: Isotr0py <Isotr0py@outlook.com> Co-authored-by: Thien Tran <gau.nernst@yahoo.com.sg> Co-authored-by: Bugen Zhao <i@bugenzhao.com> Co-authored-by: Yongye Zhu <zyy1102000@gmail.com> Co-authored-by: Ziming Huang <zelda.huanghuang@gmail.com> Co-authored-by: Roger Wang <hey@rogerw.io> Co-authored-by: Isotr0py <mozf@inferact.ai> Co-authored-by: aoshen02 <aoshen02@users.noreply.github.com> Co-authored-by: Woosuk Kwon <woosuk@inferact.ai> Co-authored-by: Jee Jee Li <jeejeelee@inferact.ai> Co-authored-by: aoshen02 <aoshen@inferact.ai> Co-authored-by: Summer Yang <girasoleyang@gmail.com> Co-authored-by: Kevin H. Luu <khluu000@gmail.com> Co-authored-by: Bowen Wang <abmfy@icloud.com> Co-authored-by: gnovack <novackgm@gmail.com> Co-authored-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai> Co-authored-by: xiaozhoupy <peiyuanzhou1994@gmail.com> Co-authored-by: Roy Wang <yasong.wang@inferact.ai> Co-authored-by: Jeff (Junze) Ma <93145857+majunze2001@users.noreply.github.com> Co-authored-by: Codex <codex@openai.com> Co-authored-by: Wei Zhao <51183510+wzhao18@users.noreply.github.com>
This commit is contained in:
co-authored by
Isotr0py
Thien Tran
Bugen Zhao
Yongye Zhu
Ziming Huang
Roger Wang
Isotr0py
aoshen02
Woosuk Kwon
Jee Jee Li
aoshen02
Summer Yang
Kevin H. Luu
Bowen Wang
gnovack
Nick Hill
Yifan Qiao
xiaozhoupy
Roy Wang
Jeff Ma
Codex
Wei Zhao
parent
0c64be8873
commit
aeeb36b1f1
@@ -0,0 +1,165 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.config import ModelConfig, ParallelConfig, SpeculativeConfig
|
||||
from vllm.transformers_utils.config import get_config
|
||||
from vllm.transformers_utils.configs.k3_dspark import K3DSparkConfig
|
||||
|
||||
|
||||
def _write_dspark_config(path, **overrides):
|
||||
path.mkdir()
|
||||
config = {
|
||||
"architectures": ["K3DSparkModel"],
|
||||
"model_type": "k3_dspark",
|
||||
"hidden_size": 7168,
|
||||
"intermediate_size": 14336,
|
||||
"num_hidden_layers": 5,
|
||||
"num_attention_heads": 64,
|
||||
"num_key_value_heads": 64,
|
||||
"q_lora_rank": 1536,
|
||||
"kv_lora_rank": 512,
|
||||
"qk_nope_head_dim": 128,
|
||||
"qk_rope_head_dim": 64,
|
||||
"v_head_dim": 128,
|
||||
"vocab_size": 163840,
|
||||
"rms_norm_eps": 1e-5,
|
||||
"max_position_embeddings": 32768,
|
||||
"rope_theta": 50000.0,
|
||||
"num_target_layers": 5,
|
||||
"target_hidden_size": 7168,
|
||||
"target_num_hidden_layers": 93,
|
||||
"target_layer_ids": [2, 23, 47, 71, 89],
|
||||
"markov_rank": 256,
|
||||
"draft_vocab_size": 163840,
|
||||
"torch_dtype": "bfloat16",
|
||||
}
|
||||
config.update(overrides)
|
||||
(path / "config.json").write_text(json.dumps(config))
|
||||
|
||||
|
||||
def _write_target_config(path):
|
||||
path.mkdir()
|
||||
config = {
|
||||
"architectures": ["LlamaForCausalLM"],
|
||||
"model_type": "llama",
|
||||
"hidden_size": 7168,
|
||||
"intermediate_size": 14336,
|
||||
"num_hidden_layers": 93,
|
||||
"num_attention_heads": 56,
|
||||
"num_key_value_heads": 8,
|
||||
"vocab_size": 163840,
|
||||
"max_position_embeddings": 32768,
|
||||
"torch_dtype": "bfloat16",
|
||||
}
|
||||
(path / "config.json").write_text(json.dumps(config))
|
||||
|
||||
|
||||
def test_dspark_mla_config_loads_from_local_json(tmp_path):
|
||||
draft_path = tmp_path / "draft"
|
||||
_write_dspark_config(draft_path)
|
||||
|
||||
config = get_config(draft_path, trust_remote_code=False)
|
||||
|
||||
assert isinstance(config, K3DSparkConfig)
|
||||
assert config.model_type == "k3_dspark"
|
||||
assert config.architectures == ["K3DSparkModel"]
|
||||
assert config.hidden_act == "silu"
|
||||
assert config.rope_parameters == {
|
||||
"rope_type": "default",
|
||||
"rope_theta": 50000.0,
|
||||
}
|
||||
assert config.n_routed_experts == 0
|
||||
assert config.draft_vocab_size == config.vocab_size
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"overrides",
|
||||
[
|
||||
{"mla_use_nope": True},
|
||||
{"mla_use_output_gate": True},
|
||||
{"mla_use_qk_norm": True},
|
||||
{"dspark_bonus_anchor": True},
|
||||
{"q_lora_rank": None},
|
||||
{"draft_vocab_size": 8192},
|
||||
{"target_layer_ids": []},
|
||||
{"num_target_layers": 4},
|
||||
],
|
||||
)
|
||||
def test_dspark_mla_rejects_unsupported_checkpoint_options(tmp_path, overrides):
|
||||
draft_path = tmp_path / "draft"
|
||||
_write_dspark_config(draft_path, **overrides)
|
||||
|
||||
with pytest.raises(ValueError, match="MLA DSpark"):
|
||||
get_config(draft_path, trust_remote_code=False)
|
||||
|
||||
|
||||
def test_dspark_mla_uses_latent_kv_geometry(tmp_path):
|
||||
draft_path = tmp_path / "draft"
|
||||
_write_dspark_config(draft_path)
|
||||
model_config = ModelConfig(
|
||||
model=str(draft_path),
|
||||
tokenizer_mode="skip",
|
||||
runner="draft",
|
||||
max_model_len=32768,
|
||||
)
|
||||
|
||||
assert model_config.is_deepseek_mla
|
||||
assert model_config.use_mla
|
||||
assert model_config.get_head_size() == 576
|
||||
# external_launcher skips ParallelConfig's local-GPU-count check so the
|
||||
# config logic can be exercised at TP8 on a single-GPU test node.
|
||||
parallel_config = ParallelConfig(
|
||||
tensor_parallel_size=8, distributed_executor_backend="external_launcher"
|
||||
)
|
||||
assert model_config.get_num_kv_heads(parallel_config) == 1
|
||||
assert model_config.get_num_attention_heads(parallel_config) == 8
|
||||
assert model_config.get_num_experts() == 0
|
||||
|
||||
|
||||
def test_dspark_mla_speculative_config_preserves_architecture(tmp_path):
|
||||
target_path = tmp_path / "target"
|
||||
draft_path = tmp_path / "draft"
|
||||
_write_target_config(target_path)
|
||||
_write_dspark_config(draft_path)
|
||||
target_config = ModelConfig(
|
||||
model=str(target_path), tokenizer_mode="skip", max_model_len=32768
|
||||
)
|
||||
speculative_config = SpeculativeConfig(
|
||||
model=str(draft_path),
|
||||
method="dspark",
|
||||
num_speculative_tokens=8,
|
||||
target_model_config=target_config,
|
||||
target_parallel_config=ParallelConfig(),
|
||||
)
|
||||
|
||||
assert speculative_config.parallel_drafting
|
||||
assert speculative_config.draft_model_config.architectures == ["K3DSparkModel"]
|
||||
assert speculative_config.draft_model_config.hf_config.model_type == "k3_dspark"
|
||||
assert speculative_config.draft_model_config.use_mla
|
||||
|
||||
|
||||
def test_dspark_mla_rejects_decode_context_parallelism(tmp_path):
|
||||
target_path = tmp_path / "target"
|
||||
draft_path = tmp_path / "draft"
|
||||
_write_target_config(target_path)
|
||||
_write_dspark_config(draft_path)
|
||||
target_config = ModelConfig(
|
||||
model=str(target_path), tokenizer_mode="skip", max_model_len=32768
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match="does not currently support decode context"):
|
||||
SpeculativeConfig(
|
||||
model=str(draft_path),
|
||||
method="dspark",
|
||||
num_speculative_tokens=8,
|
||||
target_model_config=target_config,
|
||||
target_parallel_config=ParallelConfig(
|
||||
tensor_parallel_size=2,
|
||||
decode_context_parallel_size=2,
|
||||
distributed_executor_backend="external_launcher",
|
||||
),
|
||||
)
|
||||
Reference in New Issue
Block a user