[Model] Add Kimi K3 support: model files and kernels [1/N] (#50089)
Signed-off-by: zjy0516 <riverclouds.zhu@qq.com> Signed-off-by: Isotr0py <Isotr0py@outlook.com> Signed-off-by: Bugen Zhao <i@bugenzhao.com> Signed-off-by: Yongye Zhu <zyy1102000@gmail.com> Co-authored-by: Isotr0py <Isotr0py@outlook.com> Co-authored-by: Thien Tran <gau.nernst@yahoo.com.sg> Co-authored-by: Bugen Zhao <i@bugenzhao.com> Co-authored-by: Yongye Zhu <zyy1102000@gmail.com> Co-authored-by: Ziming Huang <zelda.huanghuang@gmail.com> Co-authored-by: Roger Wang <hey@rogerw.io> Co-authored-by: Isotr0py <mozf@inferact.ai> Co-authored-by: aoshen02 <aoshen02@users.noreply.github.com> Co-authored-by: Woosuk Kwon <woosuk@inferact.ai> Co-authored-by: Jee Jee Li <jeejeelee@inferact.ai> Co-authored-by: aoshen02 <aoshen@inferact.ai> Co-authored-by: Summer Yang <girasoleyang@gmail.com> Co-authored-by: Kevin H. Luu <khluu000@gmail.com> Co-authored-by: Bowen Wang <abmfy@icloud.com> Co-authored-by: gnovack <novackgm@gmail.com> Co-authored-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai> Co-authored-by: xiaozhoupy <peiyuanzhou1994@gmail.com> Co-authored-by: Roy Wang <yasong.wang@inferact.ai> Co-authored-by: Jeff (Junze) Ma <93145857+majunze2001@users.noreply.github.com> Co-authored-by: Codex <codex@openai.com>
This commit is contained in:
co-authored by
Isotr0py
Thien Tran
Bugen Zhao
Yongye Zhu
Ziming Huang
Roger Wang
Isotr0py
aoshen02
Woosuk Kwon
Jee Jee Li
aoshen02
Summer Yang
Kevin H. Luu
Bowen Wang
gnovack
Nick Hill
Yifan Qiao
xiaozhoupy
Roy Wang
Jeff Ma
Codex
parent
6f00a1ae3b
commit
7c6729b769
@@ -197,6 +197,46 @@ def test_silu_and_mul_with_clamp(
|
||||
opcheck(torch.ops._C.silu_and_mul_with_clamp, (out_buf, x, swiglu_limit))
|
||||
|
||||
|
||||
@pytest.mark.parametrize("linear_beta", [-1.0, 2.0])
|
||||
@pytest.mark.parametrize("dtype", [torch.half, torch.bfloat16])
|
||||
@torch.inference_mode()
|
||||
def test_masked_situ_and_mul(
|
||||
default_vllm_config,
|
||||
linear_beta: float,
|
||||
dtype: torch.dtype,
|
||||
) -> None:
|
||||
"""Masked SITU computes valid expert rows and preserves padded zeros."""
|
||||
device = CUDA_DEVICES[0]
|
||||
num_experts, max_num_tokens, d = 4, 7, 512
|
||||
beta = 1.5
|
||||
input = torch.randn(num_experts, max_num_tokens, 2 * d, dtype=dtype, device=device)
|
||||
expert_num_tokens = torch.tensor([0, 1, 4, 7], dtype=torch.int32, device=device)
|
||||
output = torch.zeros(num_experts, max_num_tokens, d, dtype=dtype, device=device)
|
||||
|
||||
torch.ops._C.masked_situ_and_mul(
|
||||
output, input, expert_num_tokens, beta, linear_beta
|
||||
)
|
||||
|
||||
gate, up = input.float().chunk(2, dim=-1)
|
||||
expected = beta * torch.tanh(gate / beta) * torch.sigmoid(gate)
|
||||
if linear_beta > 0:
|
||||
up = linear_beta * torch.tanh(up / linear_beta)
|
||||
expected = (expected * up).to(dtype)
|
||||
for expert, num_tokens in enumerate(expert_num_tokens.cpu().tolist()):
|
||||
torch.testing.assert_close(
|
||||
output[expert, :num_tokens],
|
||||
expected[expert, :num_tokens],
|
||||
atol=get_default_atol(output),
|
||||
rtol=get_default_rtol(output),
|
||||
)
|
||||
assert torch.count_nonzero(output[expert, num_tokens:]) == 0
|
||||
|
||||
opcheck(
|
||||
torch.ops._C.masked_situ_and_mul,
|
||||
(output, input, expert_num_tokens, beta, linear_beta),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"activation",
|
||||
[
|
||||
|
||||
@@ -13,7 +13,7 @@ from __future__ import annotations
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.models.deepseek_v4.common.ops import fused_q_kv_rmsnorm
|
||||
from vllm.models.common.ops import fused_q_kv_rmsnorm
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
|
||||
@@ -7,7 +7,9 @@ matching the eager triton kernel output."""
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.third_party.flash_linear_attention.ops.kda import FusedRMSNormGated
|
||||
from vllm.third_party.flash_linear_attention.ops.fused_norm_gate import (
|
||||
FusedRMSNormGated,
|
||||
)
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
|
||||
DTYPES = [torch.bfloat16]
|
||||
|
||||
Reference in New Issue
Block a user