[New model] Kimi K3 (#50000)
Signed-off-by: zjy0516 <riverclouds.zhu@qq.com> Signed-off-by: Isotr0py <Isotr0py@outlook.com> Signed-off-by: Bugen Zhao <i@bugenzhao.com> Signed-off-by: Jee Jee Li <jeejeelee@inferact.ai> Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai> Signed-off-by: Yongye Zhu <zyy1102000@gmail.com> Signed-off-by: wzhao18 <wzhao18.sz@gmail.com> Co-authored-by: Isotr0py <Isotr0py@outlook.com> Co-authored-by: Thien Tran <gau.nernst@yahoo.com.sg> Co-authored-by: Bugen Zhao <i@bugenzhao.com> Co-authored-by: Yongye Zhu <zyy1102000@gmail.com> Co-authored-by: Ziming Huang <zelda.huanghuang@gmail.com> Co-authored-by: Roger Wang <hey@rogerw.io> Co-authored-by: Isotr0py <mozf@inferact.ai> Co-authored-by: aoshen02 <aoshen02@users.noreply.github.com> Co-authored-by: Woosuk Kwon <woosuk@inferact.ai> Co-authored-by: Jee Jee Li <jeejeelee@inferact.ai> Co-authored-by: aoshen02 <aoshen@inferact.ai> Co-authored-by: Summer Yang <girasoleyang@gmail.com> Co-authored-by: Kevin H. Luu <khluu000@gmail.com> Co-authored-by: Bowen Wang <abmfy@icloud.com> Co-authored-by: gnovack <novackgm@gmail.com> Co-authored-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai> Co-authored-by: xiaozhoupy <peiyuanzhou1994@gmail.com> Co-authored-by: Roy Wang <yasong.wang@inferact.ai> Co-authored-by: Jeff (Junze) Ma <93145857+majunze2001@users.noreply.github.com> Co-authored-by: Codex <codex@openai.com> Co-authored-by: Wei Zhao <51183510+wzhao18@users.noreply.github.com>
This commit is contained in:
co-authored by
Isotr0py
Thien Tran
Bugen Zhao
Yongye Zhu
Ziming Huang
Roger Wang
Isotr0py
aoshen02
Woosuk Kwon
Jee Jee Li
aoshen02
Summer Yang
Kevin H. Luu
Bowen Wang
gnovack
Nick Hill
Yifan Qiao
xiaozhoupy
Roy Wang
Jeff Ma
Codex
Wei Zhao
parent
0c64be8873
commit
aeeb36b1f1
@@ -25,6 +25,9 @@ from vllm.model_executor.layers.fused_moe.config import (
|
||||
FusedMoEQuantDesc,
|
||||
fp8_w8a8_moe_quant_config,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.deep_gemm_utils import (
|
||||
deepgemm_moe_permute,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.experts.triton_deep_gemm_moe import (
|
||||
TritonOrDeepGemmExperts,
|
||||
)
|
||||
@@ -41,6 +44,35 @@ from vllm.utils.deep_gemm import (
|
||||
BLOCK_SIZE = [128, 128]
|
||||
|
||||
|
||||
@pytest.mark.skipif(not is_deep_gemm_supported(), reason="Requires deep_gemm kernels")
|
||||
def test_deepgemm_moe_permute_initializes_padding_scales(workspace_init):
|
||||
hidden_states = torch.randn(2, 128, device="cuda", dtype=torch.bfloat16)
|
||||
activations, scales = per_token_group_quant_fp8(
|
||||
hidden_states,
|
||||
group_size=128,
|
||||
use_ue8m0=True,
|
||||
)
|
||||
topk_ids = torch.tensor([[0], [1]], device="cuda", dtype=torch.int64)
|
||||
|
||||
_, permuted_scales, expert_ids, _, _ = deepgemm_moe_permute(
|
||||
aq=activations,
|
||||
aq_scale=scales,
|
||||
topk_ids=topk_ids,
|
||||
local_num_experts=2,
|
||||
expert_map=None,
|
||||
expert_tokens_meta=None,
|
||||
)
|
||||
|
||||
padding = expert_ids < 0
|
||||
assert padding.any()
|
||||
torch.testing.assert_close(
|
||||
permuted_scales[padding],
|
||||
torch.zeros_like(permuted_scales[padding]),
|
||||
rtol=0,
|
||||
atol=0,
|
||||
)
|
||||
|
||||
|
||||
def make_block_quant_fp8_weights(
|
||||
e: int,
|
||||
n: int,
|
||||
|
||||
Reference in New Issue
Block a user