[Model] Add Kimi K3 support: model files and kernels [1/N] (#50089)
Signed-off-by: zjy0516 <riverclouds.zhu@qq.com> Signed-off-by: Isotr0py <Isotr0py@outlook.com> Signed-off-by: Bugen Zhao <i@bugenzhao.com> Signed-off-by: Yongye Zhu <zyy1102000@gmail.com> Co-authored-by: Isotr0py <Isotr0py@outlook.com> Co-authored-by: Thien Tran <gau.nernst@yahoo.com.sg> Co-authored-by: Bugen Zhao <i@bugenzhao.com> Co-authored-by: Yongye Zhu <zyy1102000@gmail.com> Co-authored-by: Ziming Huang <zelda.huanghuang@gmail.com> Co-authored-by: Roger Wang <hey@rogerw.io> Co-authored-by: Isotr0py <mozf@inferact.ai> Co-authored-by: aoshen02 <aoshen02@users.noreply.github.com> Co-authored-by: Woosuk Kwon <woosuk@inferact.ai> Co-authored-by: Jee Jee Li <jeejeelee@inferact.ai> Co-authored-by: aoshen02 <aoshen@inferact.ai> Co-authored-by: Summer Yang <girasoleyang@gmail.com> Co-authored-by: Kevin H. Luu <khluu000@gmail.com> Co-authored-by: Bowen Wang <abmfy@icloud.com> Co-authored-by: gnovack <novackgm@gmail.com> Co-authored-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai> Co-authored-by: xiaozhoupy <peiyuanzhou1994@gmail.com> Co-authored-by: Roy Wang <yasong.wang@inferact.ai> Co-authored-by: Jeff (Junze) Ma <93145857+majunze2001@users.noreply.github.com> Co-authored-by: Codex <codex@openai.com>
This commit is contained in:
co-authored by
Isotr0py
Thien Tran
Bugen Zhao
Yongye Zhu
Ziming Huang
Roger Wang
Isotr0py
aoshen02
Woosuk Kwon
Jee Jee Li
aoshen02
Summer Yang
Kevin H. Luu
Bowen Wang
gnovack
Nick Hill
Yifan Qiao
xiaozhoupy
Roy Wang
Jeff Ma
Codex
parent
6f00a1ae3b
commit
7c6729b769
@@ -23,6 +23,53 @@ from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
|
||||
|
||||
def _run_single_group_topk(
|
||||
logits: torch.Tensor,
|
||||
bias: torch.Tensor,
|
||||
topk: int,
|
||||
*,
|
||||
scoring_func: str,
|
||||
renormalize: bool,
|
||||
routed_scaling_factor: float = 1.0,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
return fused_grouped_topk(
|
||||
hidden_states=torch.empty(
|
||||
(logits.shape[0], 0), dtype=logits.dtype, device=logits.device
|
||||
),
|
||||
gating_output=logits,
|
||||
topk=topk,
|
||||
renormalize=renormalize,
|
||||
e_score_correction_bias=bias,
|
||||
num_expert_group=1,
|
||||
topk_group=1,
|
||||
scoring_func=scoring_func,
|
||||
routed_scaling_factor=routed_scaling_factor,
|
||||
)
|
||||
|
||||
|
||||
def _single_group_reference(
|
||||
logits: torch.Tensor,
|
||||
bias: torch.Tensor,
|
||||
topk: int,
|
||||
*,
|
||||
scoring_func: str,
|
||||
renormalize: bool,
|
||||
routed_scaling_factor: float = 1.0,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
if scoring_func == "sigmoid":
|
||||
scores = 0.5 * torch.tanh(0.5 * logits.float()) + 0.5
|
||||
else:
|
||||
scores = torch.softmax(logits, dim=-1).float()
|
||||
indices = torch.argsort(
|
||||
scores + bias.float(), dim=-1, descending=True, stable=True
|
||||
)[:, :topk]
|
||||
values = scores.gather(1, indices)
|
||||
if renormalize:
|
||||
values /= values.sum(dim=-1, keepdim=True) + 1e-20
|
||||
values *= routed_scaling_factor
|
||||
return values, indices.to(torch.int32)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda(), reason="This test is skipped on non-CUDA platform."
|
||||
)
|
||||
@@ -101,3 +148,222 @@ def test_grouped_topk(
|
||||
baseline_topk_weights, test_topk_weights, atol=2e-2, rtol=0
|
||||
)
|
||||
torch.testing.assert_close(baseline_topk_ids, test_topk_ids, atol=0, rtol=0)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda(), reason="This test is skipped on non-CUDA platform."
|
||||
)
|
||||
def test_grouped_topk_single_group_large_batch():
|
||||
set_random_seed(0)
|
||||
logits = torch.randn((1536, 896), dtype=torch.bfloat16, device="cuda")
|
||||
bias = torch.randn((896,), dtype=torch.float32, device="cuda")
|
||||
|
||||
expected_values, expected_ids = _single_group_reference(
|
||||
logits, bias, 16, scoring_func="sigmoid", renormalize=True
|
||||
)
|
||||
actual_values, actual_ids = _run_single_group_topk(
|
||||
logits, bias, 16, scoring_func="sigmoid", renormalize=True
|
||||
)
|
||||
|
||||
torch.testing.assert_close(actual_ids, expected_ids)
|
||||
torch.testing.assert_close(actual_values, expected_values, atol=2e-5, rtol=0)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda(), reason="This test is skipped on non-CUDA platform."
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"num_experts,topk,input_dtype,bias_dtype",
|
||||
[
|
||||
(512, 9, torch.bfloat16, torch.float32),
|
||||
(512, 16, torch.float16, torch.float16),
|
||||
(513, 9, torch.float32, torch.bfloat16),
|
||||
(513, 16, torch.bfloat16, torch.float32),
|
||||
(895, 9, torch.float16, torch.bfloat16),
|
||||
(896, 16, torch.float32, torch.float16),
|
||||
(897, 9, torch.bfloat16, torch.bfloat16),
|
||||
(897, 16, torch.float16, torch.float32),
|
||||
(1024, 9, torch.float32, torch.bfloat16),
|
||||
(1024, 16, torch.bfloat16, torch.float16),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"scoring_func,renormalize,routed_scaling_factor",
|
||||
[
|
||||
("sigmoid", True, 1.0),
|
||||
("sigmoid", False, 2.5),
|
||||
("softmax", True, 2.5),
|
||||
("softmax", False, 1.0),
|
||||
],
|
||||
)
|
||||
def test_grouped_topk_single_group_tiers(
|
||||
num_experts: int,
|
||||
topk: int,
|
||||
input_dtype: torch.dtype,
|
||||
bias_dtype: torch.dtype,
|
||||
scoring_func: str,
|
||||
renormalize: bool,
|
||||
routed_scaling_factor: float,
|
||||
):
|
||||
set_random_seed(7)
|
||||
logits = torch.randn((17, num_experts), dtype=input_dtype, device="cuda")
|
||||
bias = torch.randn((num_experts,), dtype=bias_dtype, device="cuda")
|
||||
|
||||
expected_values, expected_ids = _single_group_reference(
|
||||
logits,
|
||||
bias,
|
||||
topk,
|
||||
scoring_func=scoring_func,
|
||||
renormalize=renormalize,
|
||||
routed_scaling_factor=routed_scaling_factor,
|
||||
)
|
||||
actual_values, actual_ids = _run_single_group_topk(
|
||||
logits,
|
||||
bias,
|
||||
topk,
|
||||
scoring_func=scoring_func,
|
||||
renormalize=renormalize,
|
||||
routed_scaling_factor=routed_scaling_factor,
|
||||
)
|
||||
|
||||
torch.testing.assert_close(actual_ids, expected_ids)
|
||||
torch.testing.assert_close(actual_values, expected_values, atol=2e-5, rtol=0)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda(), reason="This test is skipped on non-CUDA platform."
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"num_experts,topk,scoring_func",
|
||||
[
|
||||
(128, 8, "sigmoid"),
|
||||
(129, 8, "sigmoid"),
|
||||
(257, 8, "sigmoid"),
|
||||
(385, 8, "sigmoid"),
|
||||
(512, 9, "sigmoid"),
|
||||
(513, 9, "sigmoid"),
|
||||
(769, 9, "sigmoid"),
|
||||
(897, 16, "sigmoid"),
|
||||
(1024, 16, "sigmoid"),
|
||||
(128, 4, "softmax"),
|
||||
(128, 5, "softmax"),
|
||||
(129, 8, "softmax"),
|
||||
(161, 8, "softmax"),
|
||||
(256, 9, "softmax"),
|
||||
(257, 8, "softmax"),
|
||||
(512, 9, "softmax"),
|
||||
(512, 17, "softmax"),
|
||||
(512, 23, "softmax"),
|
||||
(513, 8, "softmax"),
|
||||
(577, 9, "softmax"),
|
||||
(769, 9, "softmax"),
|
||||
(897, 9, "softmax"),
|
||||
(1024, 16, "softmax"),
|
||||
],
|
||||
)
|
||||
def test_grouped_topk_single_group_capacity_tiers(
|
||||
num_experts: int,
|
||||
topk: int,
|
||||
scoring_func: str,
|
||||
):
|
||||
set_random_seed(11)
|
||||
logits = torch.randn((3, num_experts), dtype=torch.bfloat16, device="cuda")
|
||||
bias = torch.randn((num_experts,), dtype=torch.float32, device="cuda")
|
||||
expected_values, expected_ids = _single_group_reference(
|
||||
logits,
|
||||
bias,
|
||||
topk,
|
||||
scoring_func=scoring_func,
|
||||
renormalize=True,
|
||||
routed_scaling_factor=2.5,
|
||||
)
|
||||
actual_values, actual_ids = _run_single_group_topk(
|
||||
logits,
|
||||
bias,
|
||||
topk,
|
||||
scoring_func=scoring_func,
|
||||
renormalize=True,
|
||||
routed_scaling_factor=2.5,
|
||||
)
|
||||
|
||||
torch.testing.assert_close(actual_ids, expected_ids)
|
||||
torch.testing.assert_close(actual_values, expected_values, atol=2e-5, rtol=0)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda(), reason="This test is skipped on non-CUDA platform."
|
||||
)
|
||||
@pytest.mark.parametrize("num_experts", [512, 896, 1024])
|
||||
def test_grouped_topk_single_group_stable_ties(num_experts: int):
|
||||
logits = torch.zeros((1, num_experts), dtype=torch.bfloat16, device="cuda")
|
||||
bias = torch.zeros((num_experts,), dtype=torch.float32, device="cuda")
|
||||
|
||||
actual_values, actual_ids = _run_single_group_topk(
|
||||
logits,
|
||||
bias,
|
||||
16,
|
||||
scoring_func="sigmoid",
|
||||
renormalize=True,
|
||||
routed_scaling_factor=2.5,
|
||||
)
|
||||
|
||||
expected_ids = torch.arange(16, dtype=torch.int32, device="cuda")[None]
|
||||
expected_values = torch.full((1, 16), 2.5 / 16, dtype=torch.float32, device="cuda")
|
||||
torch.testing.assert_close(actual_ids, expected_ids)
|
||||
torch.testing.assert_close(actual_values, expected_values, atol=2e-5, rtol=0)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda(), reason="This test is skipped on non-CUDA platform."
|
||||
)
|
||||
@pytest.mark.parametrize("num_experts", [512, 896, 1024])
|
||||
@pytest.mark.parametrize("num_finite", [0, 15])
|
||||
@pytest.mark.parametrize("renormalize", [False, True])
|
||||
def test_grouped_topk_single_group_nonfinite_scores(
|
||||
num_experts: int, num_finite: int, renormalize: bool
|
||||
):
|
||||
logits = torch.full(
|
||||
(1, num_experts), float("nan"), dtype=torch.bfloat16, device="cuda"
|
||||
)
|
||||
if num_finite:
|
||||
logits[0, :num_finite] = torch.arange(
|
||||
num_finite, dtype=torch.bfloat16, device="cuda"
|
||||
)
|
||||
logits[0, num_finite] = torch.inf
|
||||
logits[0, num_finite + 1] = -torch.inf
|
||||
bias = torch.zeros((num_experts,), dtype=torch.float32, device="cuda")
|
||||
|
||||
actual_values, actual_ids = _run_single_group_topk(
|
||||
logits,
|
||||
bias,
|
||||
16,
|
||||
scoring_func="sigmoid",
|
||||
renormalize=renormalize,
|
||||
routed_scaling_factor=2.5,
|
||||
)
|
||||
|
||||
if num_finite == 0:
|
||||
expected_ids = torch.arange(16, dtype=torch.int32, device="cuda")[None]
|
||||
if renormalize:
|
||||
expected_values = torch.full(
|
||||
(1, 16), 1 / 16, dtype=torch.float32, device="cuda"
|
||||
)
|
||||
else:
|
||||
expected_values = torch.zeros((1, 16), dtype=torch.float32, device="cuda")
|
||||
else:
|
||||
expected_ids = torch.cat(
|
||||
(
|
||||
torch.arange(num_finite - 1, -1, -1, dtype=torch.int32, device="cuda"),
|
||||
torch.tensor([num_finite], dtype=torch.int32, device="cuda"),
|
||||
)
|
||||
)[None]
|
||||
finite_values = logits[0, :num_finite].float().sigmoid().flip(0)
|
||||
if renormalize:
|
||||
finite_values /= finite_values.sum()
|
||||
finite_values *= 2.5
|
||||
expected_values = torch.cat(
|
||||
(finite_values, torch.zeros(1, dtype=torch.float32, device="cuda"))
|
||||
)[None]
|
||||
|
||||
torch.testing.assert_close(actual_ids, expected_ids)
|
||||
torch.testing.assert_close(actual_values, expected_values, atol=2e-5, rtol=0)
|
||||
|
||||
@@ -269,6 +269,7 @@ def test_moe_align_block_size_with_expert_map(
|
||||
if (experts[k] in local_experts) or not mask_inactive_experts
|
||||
else -1
|
||||
)
|
||||
topk_ids[0, 0] = -1
|
||||
|
||||
actual_sorted_ids, actual_expert_ids, actual_num_tokens = moe_align_block_size(
|
||||
topk_ids=topk_ids,
|
||||
|
||||
Reference in New Issue
Block a user