[Model] Add Kimi K3 support: model files and kernels [1/N] (#50089)

Signed-off-by: zjy0516 <riverclouds.zhu@qq.com>
Signed-off-by: Isotr0py <Isotr0py@outlook.com>
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
Signed-off-by: Yongye Zhu <zyy1102000@gmail.com>
Co-authored-by: Isotr0py <Isotr0py@outlook.com>
Co-authored-by: Thien Tran <gau.nernst@yahoo.com.sg>
Co-authored-by: Bugen Zhao <i@bugenzhao.com>
Co-authored-by: Yongye Zhu <zyy1102000@gmail.com>
Co-authored-by: Ziming Huang <zelda.huanghuang@gmail.com>
Co-authored-by: Roger Wang <hey@rogerw.io>
Co-authored-by: Isotr0py <mozf@inferact.ai>
Co-authored-by: aoshen02 <aoshen02@users.noreply.github.com>
Co-authored-by: Woosuk Kwon <woosuk@inferact.ai>
Co-authored-by: Jee Jee Li <jeejeelee@inferact.ai>
Co-authored-by: aoshen02 <aoshen@inferact.ai>
Co-authored-by: Summer Yang <girasoleyang@gmail.com>
Co-authored-by: Kevin H. Luu <khluu000@gmail.com>
Co-authored-by: Bowen Wang <abmfy@icloud.com>
Co-authored-by: gnovack <novackgm@gmail.com>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai>
Co-authored-by: xiaozhoupy <peiyuanzhou1994@gmail.com>
Co-authored-by: Roy Wang <yasong.wang@inferact.ai>
Co-authored-by: Jeff (Junze) Ma <93145857+majunze2001@users.noreply.github.com>
Co-authored-by: Codex <codex@openai.com>
This commit is contained in:
Jiangyun Zhu
2026-07-29 14:10:58 +08:00
committed by GitHub
co-authored by Isotr0py Thien Tran Bugen Zhao Yongye Zhu Ziming Huang Roger Wang Isotr0py aoshen02 Woosuk Kwon Jee Jee Li aoshen02 Summer Yang Kevin H. Luu Bowen Wang gnovack Nick Hill Yifan Qiao xiaozhoupy Roy Wang Jeff Ma Codex
parent 6f00a1ae3b
commit 7c6729b769
109 changed files with 28028 additions and 792 deletions
@@ -0,0 +1,73 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
from vllm.distributed import (
get_tensor_model_parallel_rank,
get_tensor_model_parallel_world_size,
get_tp_group,
tensor_model_parallel_all_gather,
tensor_model_parallel_reduce_scatter,
)
def _custom_collective(
name: str,
x: torch.Tensor,
) -> torch.Tensor | None:
device_communicator = get_tp_group().device_communicator
if device_communicator is None:
return None
collective = getattr(device_communicator, name, None)
if collective is None:
return None
return collective(x)
def sp_all_gather(x: torch.Tensor) -> torch.Tensor:
output = _custom_collective("custom_all_gather", x)
if output is not None:
return output
return tensor_model_parallel_all_gather(x, 0)
def sp_reduce_scatter(x: torch.Tensor) -> torch.Tensor:
assert x.ndim == 2
tp_size = get_tensor_model_parallel_world_size()
sp_pad = (-x.shape[0]) % tp_size
if sp_pad > 0:
x = torch.nn.functional.pad(x, (0, 0, 0, sp_pad))
output = _custom_collective("custom_reduce_scatter", x)
if output is not None:
return output
return tensor_model_parallel_reduce_scatter(x, 0)
def sp_shard(x: torch.Tensor) -> torch.Tensor:
assert x.ndim == 2
tp_size = get_tensor_model_parallel_world_size()
tp_rank = get_tensor_model_parallel_rank()
sp_pad = (-x.shape[0]) % tp_size
if sp_pad > 0:
x = torch.nn.functional.pad(x, (0, 0, 0, sp_pad))
chunk = x.shape[0] // tp_size
return x[tp_rank * chunk : (tp_rank + 1) * chunk]
def sp_padding_mask(
is_padding: torch.Tensor | None,
hidden_states: torch.Tensor,
) -> torch.Tensor:
num_tokens = hidden_states.shape[0]
if is_padding is None:
is_padding = hidden_states.new_zeros(num_tokens, dtype=torch.bool)
assert is_padding.shape[0] == num_tokens
tp_size = get_tensor_model_parallel_world_size()
sp_pad = (-num_tokens) % tp_size
if sp_pad > 0:
is_padding = torch.nn.functional.pad(is_padding, (0, sp_pad), value=True)
chunk = is_padding.shape[0] // tp_size
tp_rank = get_tensor_model_parallel_rank()
return is_padding[tp_rank * chunk : (tp_rank + 1) * chunk]