[Model] Add Kimi K3 support: model files and kernels [1/N] (#50089)
Signed-off-by: zjy0516 <riverclouds.zhu@qq.com> Signed-off-by: Isotr0py <Isotr0py@outlook.com> Signed-off-by: Bugen Zhao <i@bugenzhao.com> Signed-off-by: Yongye Zhu <zyy1102000@gmail.com> Co-authored-by: Isotr0py <Isotr0py@outlook.com> Co-authored-by: Thien Tran <gau.nernst@yahoo.com.sg> Co-authored-by: Bugen Zhao <i@bugenzhao.com> Co-authored-by: Yongye Zhu <zyy1102000@gmail.com> Co-authored-by: Ziming Huang <zelda.huanghuang@gmail.com> Co-authored-by: Roger Wang <hey@rogerw.io> Co-authored-by: Isotr0py <mozf@inferact.ai> Co-authored-by: aoshen02 <aoshen02@users.noreply.github.com> Co-authored-by: Woosuk Kwon <woosuk@inferact.ai> Co-authored-by: Jee Jee Li <jeejeelee@inferact.ai> Co-authored-by: aoshen02 <aoshen@inferact.ai> Co-authored-by: Summer Yang <girasoleyang@gmail.com> Co-authored-by: Kevin H. Luu <khluu000@gmail.com> Co-authored-by: Bowen Wang <abmfy@icloud.com> Co-authored-by: gnovack <novackgm@gmail.com> Co-authored-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai> Co-authored-by: xiaozhoupy <peiyuanzhou1994@gmail.com> Co-authored-by: Roy Wang <yasong.wang@inferact.ai> Co-authored-by: Jeff (Junze) Ma <93145857+majunze2001@users.noreply.github.com> Co-authored-by: Codex <codex@openai.com>
This commit is contained in:
co-authored by
Isotr0py
Thien Tran
Bugen Zhao
Yongye Zhu
Ziming Huang
Roger Wang
Isotr0py
aoshen02
Woosuk Kwon
Jee Jee Li
aoshen02
Summer Yang
Kevin H. Luu
Bowen Wang
gnovack
Nick Hill
Yifan Qiao
xiaozhoupy
Roy Wang
Jeff Ma
Codex
parent
6f00a1ae3b
commit
7c6729b769
@@ -1080,6 +1080,7 @@ def causal_conv1d_update(
|
||||
block_idx_last_scheduled_token: torch.Tensor | None = None,
|
||||
initial_state_idx: torch.Tensor | None = None,
|
||||
validate_data=False,
|
||||
out: torch.Tensor | None = None,
|
||||
):
|
||||
"""
|
||||
x: Input tensor which can take the following shapes:
|
||||
@@ -1117,7 +1118,8 @@ def causal_conv1d_update(
|
||||
for example: conv_state_indices = [null_block_id, 1, 20, null_block_id]
|
||||
in this case, the kernel will not process entries at
|
||||
indices 0 and 3
|
||||
out: (batch, dim) or (batch, dim, seqlen) or (num_tokens, dim), same shape as `x`
|
||||
out: optional output tensor with the same shape as `x`. When omitted,
|
||||
the input is overwritten.
|
||||
"""
|
||||
if validate_data:
|
||||
assert null_block_id is not None
|
||||
@@ -1129,10 +1131,22 @@ def causal_conv1d_update(
|
||||
|
||||
original_x_dtype = x.dtype
|
||||
x = x.to(conv_state.dtype)
|
||||
if out is None:
|
||||
out = x
|
||||
else:
|
||||
if out.shape != x.shape:
|
||||
raise ValueError(
|
||||
f"`out` shape {tuple(out.shape)} must match `x` shape {tuple(x.shape)}."
|
||||
)
|
||||
if out.dtype != original_x_dtype or out.device != x.device:
|
||||
raise ValueError(
|
||||
"`out` must have the same dtype and device as the input `x`."
|
||||
)
|
||||
unsqueeze = query_start_loc is None and x.dim() == 2
|
||||
if unsqueeze:
|
||||
# make it (batch, dim, seqlen) with seqlen == 1
|
||||
x = x.unsqueeze(-1)
|
||||
out = out.unsqueeze(-1)
|
||||
if query_start_loc is None:
|
||||
batch, dim, seqlen = x.shape
|
||||
else:
|
||||
@@ -1159,8 +1173,6 @@ def causal_conv1d_update(
|
||||
assert num_cache_lines >= batch
|
||||
assert weight.stride(1) == 1 # Need this
|
||||
|
||||
# adopt the strategy in vLLM that overwrite on 'x' directly, rather than creating a new tensor 'o'
|
||||
out = x
|
||||
stride_w_dim, stride_w_width = weight.stride()
|
||||
|
||||
if query_start_loc is None:
|
||||
|
||||
@@ -0,0 +1,83 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import tl, triton
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _gather_initial_states_kernel(
|
||||
state_ptr,
|
||||
indices_ptr,
|
||||
has_initial_state_ptr,
|
||||
output_ptr,
|
||||
stride_state_batch,
|
||||
stride_indices,
|
||||
stride_has_initial_state,
|
||||
row_size: tl.constexpr,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
launch_pdl: tl.constexpr,
|
||||
):
|
||||
block_idx = tl.program_id(0)
|
||||
batch_idx = tl.program_id(1)
|
||||
offsets = block_idx * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)
|
||||
mask = offsets < row_size
|
||||
|
||||
if launch_pdl:
|
||||
tl.extra.cuda.gdc_wait()
|
||||
tl.extra.cuda.gdc_launch_dependents()
|
||||
|
||||
has_initial_state = tl.load(
|
||||
has_initial_state_ptr + batch_idx * stride_has_initial_state
|
||||
).to(tl.int1)
|
||||
state_idx = tl.load(indices_ptr + batch_idx * stride_indices).to(tl.int64)
|
||||
state_idx = tl.where(has_initial_state, state_idx, 0)
|
||||
values = tl.load(
|
||||
state_ptr + state_idx * stride_state_batch + offsets,
|
||||
mask=mask & has_initial_state,
|
||||
other=0.0,
|
||||
)
|
||||
tl.store(output_ptr + batch_idx * row_size + offsets, values, mask=mask)
|
||||
|
||||
|
||||
def gather_initial_states(
|
||||
state: torch.Tensor,
|
||||
indices: torch.Tensor,
|
||||
has_initial_state: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
"""Gather dense state rows, replacing uninitialized rows with zeros."""
|
||||
assert state.ndim >= 2
|
||||
assert state.is_cuda
|
||||
assert indices.ndim == 1 and has_initial_state.ndim == 1
|
||||
assert indices.shape == has_initial_state.shape
|
||||
assert indices.device == state.device
|
||||
assert has_initial_state.device == state.device
|
||||
assert indices.dtype in (torch.int32, torch.int64)
|
||||
assert has_initial_state.dtype == torch.bool
|
||||
|
||||
row_size = state[0].numel()
|
||||
# Mamba pages may pad stride(0), but each state row remains dense.
|
||||
assert state[0].is_contiguous()
|
||||
output = torch.empty(
|
||||
(indices.numel(), *state.shape[1:]),
|
||||
dtype=state.dtype,
|
||||
device=state.device,
|
||||
)
|
||||
block_size = min(triton.next_power_of_2(row_size), 1024)
|
||||
grid = (triton.cdiv(row_size, block_size), indices.numel())
|
||||
_gather_initial_states_kernel[grid](
|
||||
state,
|
||||
indices,
|
||||
has_initial_state,
|
||||
output,
|
||||
state.stride(0),
|
||||
indices.stride(0),
|
||||
has_initial_state.stride(0),
|
||||
row_size=row_size,
|
||||
BLOCK_SIZE=block_size,
|
||||
num_warps=8,
|
||||
launch_pdl=current_platform.is_arch_support_pdl(),
|
||||
)
|
||||
return output
|
||||
Reference in New Issue
Block a user