Files
+12 7c6729b769 [Model] Add Kimi K3 support: model files and kernels [1/N] (#50089)
Signed-off-by: zjy0516 <riverclouds.zhu@qq.com>
Signed-off-by: Isotr0py <Isotr0py@outlook.com>
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
Signed-off-by: Yongye Zhu <zyy1102000@gmail.com>
Co-authored-by: Isotr0py <Isotr0py@outlook.com>
Co-authored-by: Thien Tran <gau.nernst@yahoo.com.sg>
Co-authored-by: Bugen Zhao <i@bugenzhao.com>
Co-authored-by: Yongye Zhu <zyy1102000@gmail.com>
Co-authored-by: Ziming Huang <zelda.huanghuang@gmail.com>
Co-authored-by: Roger Wang <hey@rogerw.io>
Co-authored-by: Isotr0py <mozf@inferact.ai>
Co-authored-by: aoshen02 <aoshen02@users.noreply.github.com>
Co-authored-by: Woosuk Kwon <woosuk@inferact.ai>
Co-authored-by: Jee Jee Li <jeejeelee@inferact.ai>
Co-authored-by: aoshen02 <aoshen@inferact.ai>
Co-authored-by: Summer Yang <girasoleyang@gmail.com>
Co-authored-by: Kevin H. Luu <khluu000@gmail.com>
Co-authored-by: Bowen Wang <abmfy@icloud.com>
Co-authored-by: gnovack <novackgm@gmail.com>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai>
Co-authored-by: xiaozhoupy <peiyuanzhou1994@gmail.com>
Co-authored-by: Roy Wang <yasong.wang@inferact.ai>
Co-authored-by: Jeff (Junze) Ma <93145857+majunze2001@users.noreply.github.com>
Co-authored-by: Codex <codex@openai.com>
2026-07-29 14:10:58 +08:00

97 lines
2.6 KiB
Python

# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Fused Kimi-K3 MTP input preparation."""
import torch
from vllm.triton_utils import tl, triton
@triton.jit
def _rms_norm(x, weight, eps, hidden_size: tl.constexpr):
x = x.to(tl.float32)
variance = tl.sum(x * x, axis=0) / hidden_size
return x * tl.rsqrt(variance + eps) * weight.to(tl.float32)
@triton.jit
def _fused_mtp_input_kernel(
positions_ptr,
inputs_embeds_ptr,
previous_hidden_states_ptr,
enorm_weight_ptr,
hnorm_weight_ptr,
output_ptr,
eps,
inputs_embeds_stride,
previous_hidden_states_stride,
output_stride,
hidden_size: tl.constexpr,
block_size: tl.constexpr,
):
token_idx = tl.program_id(0).to(tl.int64)
input_idx = tl.program_id(1)
offsets = tl.arange(0, block_size)
mask = offsets < hidden_size
if input_idx == 0:
position = tl.load(positions_ptr + token_idx)
values = tl.load(
inputs_embeds_ptr + token_idx * inputs_embeds_stride + offsets,
mask=mask & (position != 0),
other=0.0,
)
weight = tl.load(enorm_weight_ptr + offsets, mask=mask, other=0.0)
else:
values = tl.load(
previous_hidden_states_ptr
+ token_idx * previous_hidden_states_stride
+ offsets,
mask=mask,
other=0.0,
)
weight = tl.load(hnorm_weight_ptr + offsets, mask=mask, other=0.0)
output = _rms_norm(values, weight, eps, hidden_size)
tl.store(
output_ptr + token_idx * output_stride + input_idx * hidden_size + offsets,
output,
mask=mask,
)
def fused_mtp_input(
positions: torch.Tensor,
inputs_embeds: torch.Tensor,
previous_hidden_states: torch.Tensor,
enorm_weight: torch.Tensor,
hnorm_weight: torch.Tensor,
eps: float,
) -> torch.Tensor:
"""Mask and normalize both MTP inputs into the projection layout."""
num_tokens, hidden_size = inputs_embeds.shape
output = torch.empty(
num_tokens,
2 * hidden_size,
dtype=inputs_embeds.dtype,
device=inputs_embeds.device,
)
if num_tokens == 0:
return output
_fused_mtp_input_kernel[(num_tokens, 2)](
positions,
inputs_embeds,
previous_hidden_states,
enorm_weight,
hnorm_weight,
output,
eps,
inputs_embeds.stride(0),
previous_hidden_states.stride(0),
output.stride(0),
hidden_size,
triton.next_power_of_2(hidden_size),
)
return output