[Model] Add Kimi K3 support: model files and kernels [1/N] (#50089)

Signed-off-by: zjy0516 <riverclouds.zhu@qq.com>
Signed-off-by: Isotr0py <Isotr0py@outlook.com>
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
Signed-off-by: Yongye Zhu <zyy1102000@gmail.com>
Co-authored-by: Isotr0py <Isotr0py@outlook.com>
Co-authored-by: Thien Tran <gau.nernst@yahoo.com.sg>
Co-authored-by: Bugen Zhao <i@bugenzhao.com>
Co-authored-by: Yongye Zhu <zyy1102000@gmail.com>
Co-authored-by: Ziming Huang <zelda.huanghuang@gmail.com>
Co-authored-by: Roger Wang <hey@rogerw.io>
Co-authored-by: Isotr0py <mozf@inferact.ai>
Co-authored-by: aoshen02 <aoshen02@users.noreply.github.com>
Co-authored-by: Woosuk Kwon <woosuk@inferact.ai>
Co-authored-by: Jee Jee Li <jeejeelee@inferact.ai>
Co-authored-by: aoshen02 <aoshen@inferact.ai>
Co-authored-by: Summer Yang <girasoleyang@gmail.com>
Co-authored-by: Kevin H. Luu <khluu000@gmail.com>
Co-authored-by: Bowen Wang <abmfy@icloud.com>
Co-authored-by: gnovack <novackgm@gmail.com>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai>
Co-authored-by: xiaozhoupy <peiyuanzhou1994@gmail.com>
Co-authored-by: Roy Wang <yasong.wang@inferact.ai>
Co-authored-by: Jeff (Junze) Ma <93145857+majunze2001@users.noreply.github.com>
Co-authored-by: Codex <codex@openai.com>
This commit is contained in:
Jiangyun Zhu
2026-07-29 14:10:58 +08:00
committed by GitHub
co-authored by Isotr0py Thien Tran Bugen Zhao Yongye Zhu Ziming Huang Roger Wang Isotr0py aoshen02 Woosuk Kwon Jee Jee Li aoshen02 Summer Yang Kevin H. Luu Bowen Wang gnovack Nick Hill Yifan Qiao xiaozhoupy Roy Wang Jeff Ma Codex
parent 6f00a1ae3b
commit 7c6729b769
109 changed files with 28028 additions and 792 deletions
+266
View File
@@ -23,6 +23,53 @@ from vllm.platforms import current_platform
from vllm.utils.torch_utils import set_random_seed
def _run_single_group_topk(
logits: torch.Tensor,
bias: torch.Tensor,
topk: int,
*,
scoring_func: str,
renormalize: bool,
routed_scaling_factor: float = 1.0,
) -> tuple[torch.Tensor, torch.Tensor]:
return fused_grouped_topk(
hidden_states=torch.empty(
(logits.shape[0], 0), dtype=logits.dtype, device=logits.device
),
gating_output=logits,
topk=topk,
renormalize=renormalize,
e_score_correction_bias=bias,
num_expert_group=1,
topk_group=1,
scoring_func=scoring_func,
routed_scaling_factor=routed_scaling_factor,
)
def _single_group_reference(
logits: torch.Tensor,
bias: torch.Tensor,
topk: int,
*,
scoring_func: str,
renormalize: bool,
routed_scaling_factor: float = 1.0,
) -> tuple[torch.Tensor, torch.Tensor]:
if scoring_func == "sigmoid":
scores = 0.5 * torch.tanh(0.5 * logits.float()) + 0.5
else:
scores = torch.softmax(logits, dim=-1).float()
indices = torch.argsort(
scores + bias.float(), dim=-1, descending=True, stable=True
)[:, :topk]
values = scores.gather(1, indices)
if renormalize:
values /= values.sum(dim=-1, keepdim=True) + 1e-20
values *= routed_scaling_factor
return values, indices.to(torch.int32)
@pytest.mark.skipif(
not current_platform.is_cuda(), reason="This test is skipped on non-CUDA platform."
)
@@ -101,3 +148,222 @@ def test_grouped_topk(
baseline_topk_weights, test_topk_weights, atol=2e-2, rtol=0
)
torch.testing.assert_close(baseline_topk_ids, test_topk_ids, atol=0, rtol=0)
@pytest.mark.skipif(
not current_platform.is_cuda(), reason="This test is skipped on non-CUDA platform."
)
def test_grouped_topk_single_group_large_batch():
set_random_seed(0)
logits = torch.randn((1536, 896), dtype=torch.bfloat16, device="cuda")
bias = torch.randn((896,), dtype=torch.float32, device="cuda")
expected_values, expected_ids = _single_group_reference(
logits, bias, 16, scoring_func="sigmoid", renormalize=True
)
actual_values, actual_ids = _run_single_group_topk(
logits, bias, 16, scoring_func="sigmoid", renormalize=True
)
torch.testing.assert_close(actual_ids, expected_ids)
torch.testing.assert_close(actual_values, expected_values, atol=2e-5, rtol=0)
@pytest.mark.skipif(
not current_platform.is_cuda(), reason="This test is skipped on non-CUDA platform."
)
@pytest.mark.parametrize(
"num_experts,topk,input_dtype,bias_dtype",
[
(512, 9, torch.bfloat16, torch.float32),
(512, 16, torch.float16, torch.float16),
(513, 9, torch.float32, torch.bfloat16),
(513, 16, torch.bfloat16, torch.float32),
(895, 9, torch.float16, torch.bfloat16),
(896, 16, torch.float32, torch.float16),
(897, 9, torch.bfloat16, torch.bfloat16),
(897, 16, torch.float16, torch.float32),
(1024, 9, torch.float32, torch.bfloat16),
(1024, 16, torch.bfloat16, torch.float16),
],
)
@pytest.mark.parametrize(
"scoring_func,renormalize,routed_scaling_factor",
[
("sigmoid", True, 1.0),
("sigmoid", False, 2.5),
("softmax", True, 2.5),
("softmax", False, 1.0),
],
)
def test_grouped_topk_single_group_tiers(
num_experts: int,
topk: int,
input_dtype: torch.dtype,
bias_dtype: torch.dtype,
scoring_func: str,
renormalize: bool,
routed_scaling_factor: float,
):
set_random_seed(7)
logits = torch.randn((17, num_experts), dtype=input_dtype, device="cuda")
bias = torch.randn((num_experts,), dtype=bias_dtype, device="cuda")
expected_values, expected_ids = _single_group_reference(
logits,
bias,
topk,
scoring_func=scoring_func,
renormalize=renormalize,
routed_scaling_factor=routed_scaling_factor,
)
actual_values, actual_ids = _run_single_group_topk(
logits,
bias,
topk,
scoring_func=scoring_func,
renormalize=renormalize,
routed_scaling_factor=routed_scaling_factor,
)
torch.testing.assert_close(actual_ids, expected_ids)
torch.testing.assert_close(actual_values, expected_values, atol=2e-5, rtol=0)
@pytest.mark.skipif(
not current_platform.is_cuda(), reason="This test is skipped on non-CUDA platform."
)
@pytest.mark.parametrize(
"num_experts,topk,scoring_func",
[
(128, 8, "sigmoid"),
(129, 8, "sigmoid"),
(257, 8, "sigmoid"),
(385, 8, "sigmoid"),
(512, 9, "sigmoid"),
(513, 9, "sigmoid"),
(769, 9, "sigmoid"),
(897, 16, "sigmoid"),
(1024, 16, "sigmoid"),
(128, 4, "softmax"),
(128, 5, "softmax"),
(129, 8, "softmax"),
(161, 8, "softmax"),
(256, 9, "softmax"),
(257, 8, "softmax"),
(512, 9, "softmax"),
(512, 17, "softmax"),
(512, 23, "softmax"),
(513, 8, "softmax"),
(577, 9, "softmax"),
(769, 9, "softmax"),
(897, 9, "softmax"),
(1024, 16, "softmax"),
],
)
def test_grouped_topk_single_group_capacity_tiers(
num_experts: int,
topk: int,
scoring_func: str,
):
set_random_seed(11)
logits = torch.randn((3, num_experts), dtype=torch.bfloat16, device="cuda")
bias = torch.randn((num_experts,), dtype=torch.float32, device="cuda")
expected_values, expected_ids = _single_group_reference(
logits,
bias,
topk,
scoring_func=scoring_func,
renormalize=True,
routed_scaling_factor=2.5,
)
actual_values, actual_ids = _run_single_group_topk(
logits,
bias,
topk,
scoring_func=scoring_func,
renormalize=True,
routed_scaling_factor=2.5,
)
torch.testing.assert_close(actual_ids, expected_ids)
torch.testing.assert_close(actual_values, expected_values, atol=2e-5, rtol=0)
@pytest.mark.skipif(
not current_platform.is_cuda(), reason="This test is skipped on non-CUDA platform."
)
@pytest.mark.parametrize("num_experts", [512, 896, 1024])
def test_grouped_topk_single_group_stable_ties(num_experts: int):
logits = torch.zeros((1, num_experts), dtype=torch.bfloat16, device="cuda")
bias = torch.zeros((num_experts,), dtype=torch.float32, device="cuda")
actual_values, actual_ids = _run_single_group_topk(
logits,
bias,
16,
scoring_func="sigmoid",
renormalize=True,
routed_scaling_factor=2.5,
)
expected_ids = torch.arange(16, dtype=torch.int32, device="cuda")[None]
expected_values = torch.full((1, 16), 2.5 / 16, dtype=torch.float32, device="cuda")
torch.testing.assert_close(actual_ids, expected_ids)
torch.testing.assert_close(actual_values, expected_values, atol=2e-5, rtol=0)
@pytest.mark.skipif(
not current_platform.is_cuda(), reason="This test is skipped on non-CUDA platform."
)
@pytest.mark.parametrize("num_experts", [512, 896, 1024])
@pytest.mark.parametrize("num_finite", [0, 15])
@pytest.mark.parametrize("renormalize", [False, True])
def test_grouped_topk_single_group_nonfinite_scores(
num_experts: int, num_finite: int, renormalize: bool
):
logits = torch.full(
(1, num_experts), float("nan"), dtype=torch.bfloat16, device="cuda"
)
if num_finite:
logits[0, :num_finite] = torch.arange(
num_finite, dtype=torch.bfloat16, device="cuda"
)
logits[0, num_finite] = torch.inf
logits[0, num_finite + 1] = -torch.inf
bias = torch.zeros((num_experts,), dtype=torch.float32, device="cuda")
actual_values, actual_ids = _run_single_group_topk(
logits,
bias,
16,
scoring_func="sigmoid",
renormalize=renormalize,
routed_scaling_factor=2.5,
)
if num_finite == 0:
expected_ids = torch.arange(16, dtype=torch.int32, device="cuda")[None]
if renormalize:
expected_values = torch.full(
(1, 16), 1 / 16, dtype=torch.float32, device="cuda"
)
else:
expected_values = torch.zeros((1, 16), dtype=torch.float32, device="cuda")
else:
expected_ids = torch.cat(
(
torch.arange(num_finite - 1, -1, -1, dtype=torch.int32, device="cuda"),
torch.tensor([num_finite], dtype=torch.int32, device="cuda"),
)
)[None]
finite_values = logits[0, :num_finite].float().sigmoid().flip(0)
if renormalize:
finite_values /= finite_values.sum()
finite_values *= 2.5
expected_values = torch.cat(
(finite_values, torch.zeros(1, dtype=torch.float32, device="cuda"))
)[None]
torch.testing.assert_close(actual_ids, expected_ids)
torch.testing.assert_close(actual_values, expected_values, atol=2e-5, rtol=0)
@@ -269,6 +269,7 @@ def test_moe_align_block_size_with_expert_map(
if (experts[k] in local_experts) or not mask_inactive_experts
else -1
)
topk_ids[0, 0] = -1
actual_sorted_ids, actual_expert_ids, actual_num_tokens = moe_align_block_size(
topk_ids=topk_ids,