[Model] Add Kimi K3 support: model files and kernels [1/N] (#50089)

Signed-off-by: zjy0516 <riverclouds.zhu@qq.com>
Signed-off-by: Isotr0py <Isotr0py@outlook.com>
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
Signed-off-by: Yongye Zhu <zyy1102000@gmail.com>
Co-authored-by: Isotr0py <Isotr0py@outlook.com>
Co-authored-by: Thien Tran <gau.nernst@yahoo.com.sg>
Co-authored-by: Bugen Zhao <i@bugenzhao.com>
Co-authored-by: Yongye Zhu <zyy1102000@gmail.com>
Co-authored-by: Ziming Huang <zelda.huanghuang@gmail.com>
Co-authored-by: Roger Wang <hey@rogerw.io>
Co-authored-by: Isotr0py <mozf@inferact.ai>
Co-authored-by: aoshen02 <aoshen02@users.noreply.github.com>
Co-authored-by: Woosuk Kwon <woosuk@inferact.ai>
Co-authored-by: Jee Jee Li <jeejeelee@inferact.ai>
Co-authored-by: aoshen02 <aoshen@inferact.ai>
Co-authored-by: Summer Yang <girasoleyang@gmail.com>
Co-authored-by: Kevin H. Luu <khluu000@gmail.com>
Co-authored-by: Bowen Wang <abmfy@icloud.com>
Co-authored-by: gnovack <novackgm@gmail.com>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai>
Co-authored-by: xiaozhoupy <peiyuanzhou1994@gmail.com>
Co-authored-by: Roy Wang <yasong.wang@inferact.ai>
Co-authored-by: Jeff (Junze) Ma <93145857+majunze2001@users.noreply.github.com>
Co-authored-by: Codex <codex@openai.com>
This commit is contained in:
Jiangyun Zhu
2026-07-29 14:10:58 +08:00
committed by GitHub
co-authored by Isotr0py Thien Tran Bugen Zhao Yongye Zhu Ziming Huang Roger Wang Isotr0py aoshen02 Woosuk Kwon Jee Jee Li aoshen02 Summer Yang Kevin H. Luu Bowen Wang gnovack Nick Hill Yifan Qiao xiaozhoupy Roy Wang Jeff Ma Codex
parent 6f00a1ae3b
commit 7c6729b769
109 changed files with 28028 additions and 792 deletions
@@ -0,0 +1,180 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from __future__ import annotations
import cutlass
import cutlass.cute as cute
from cuda.bindings.driver import CUstream
from cutlass import const_expr
class CuteSkinnyGemm:
"""Shape-dynamic low-latency GEMM for small token counts.
Computes ``C[M, N] = A[M, K] @ B[N, K].T + residual`` with BF16 or FP16
inputs, FP32 accumulators, and an output matching the input dtype. The
residual term is optional and is added before the output conversion. N and
K are runtime values. The tiny M dimension is fully unrolled alongside a
small set of tuning parameters.
"""
def __init__(
self,
*,
element_type,
num_rows: int,
block_size: int,
outputs_per_block: int,
vector_width: int = 8,
k_unroll: int = 1,
has_residual: bool = False,
use_pdl: bool = False,
) -> None:
if block_size % cute.arch.WARP_SIZE != 0:
raise ValueError("block_size must be a multiple of the warp size")
self.element_type = element_type
self.num_rows = num_rows
self.block_size = block_size
self.outputs_per_block = outputs_per_block
self.vector_width = vector_width
self.k_unroll = k_unroll
self.has_residual = has_residual
self.use_pdl = use_pdl
self.num_warps = block_size // cute.arch.WARP_SIZE
@cute.jit
def __call__(
self,
gA: cute.Tensor,
gB: cute.Tensor,
gResidual: cute.Tensor,
gC: cute.Tensor,
stream: CUstream,
) -> None:
n = cute.size(gB, mode=[0])
k = cute.size(gA, mode=[1])
copy_a = cute.make_copy_atom(
cute.nvgpu.CopyG2ROp(),
self.element_type,
num_bits_per_copy=self.vector_width * self.element_type.width,
load_cache_mode=cute.nvgpu.LoadCacheMode.ALWAYS,
)
copy_b = cute.make_copy_atom(
cute.nvgpu.CopyG2ROp(),
self.element_type,
num_bits_per_copy=self.vector_width * self.element_type.width,
load_cache_mode=cute.nvgpu.LoadCacheMode.STREAMING,
)
self.kernel(gA, gB, gResidual, gC, k, copy_a, copy_b).launch(
grid=[cute.ceil_div(n, self.outputs_per_block), 1, 1],
block=[self.block_size, 1, 1],
smem=self.num_rows * self.outputs_per_block * self.num_warps * 4,
stream=stream,
use_pdl=self.use_pdl,
min_blocks_per_mp=1,
)
@cute.kernel
def kernel(
self,
gA: cute.Tensor,
gB: cute.Tensor,
gResidual: cute.Tensor,
gC: cute.Tensor,
k_extent: cutlass.Int32,
copy_a: cute.CopyAtom,
copy_b: cute.CopyAtom,
) -> None:
tidx, _, _ = cute.arch.thread_idx()
block_idx, _, _ = cute.arch.block_idx()
warp_idx = cute.arch.warp_idx()
num_rows: cutlass.Constexpr = self.num_rows
outputs_per_block: cutlass.Constexpr = self.outputs_per_block
vector_width: cutlass.Constexpr = self.vector_width
block_size: cutlass.Constexpr = self.block_size
num_warps: cutlass.Constexpr = self.num_warps
acc_layout = cute.make_layout(
(num_rows, outputs_per_block), stride=(outputs_per_block, 1)
)
acc = cute.make_rmem_tensor(acc_layout, cutlass.Float32)
acc.fill(0.0)
if const_expr(self.use_pdl):
cute.arch.griddepcontrol_wait()
n_base = block_idx * outputs_per_block
k_tile_size: cutlass.Constexpr = block_size * vector_width
num_k_tiles = k_extent // k_tile_size
gA_vec = cute.logical_divide(gA, (None, vector_width))
gB_vec = cute.logical_divide(gB, (None, vector_width))
# Layout after both divides is (M/N, K_TILE, K_LANE, K_VEC).
tA_all = cute.logical_divide(gA_vec, (None, (None, block_size)))
tB_all = cute.logical_divide(gB_vec, (None, (None, block_size)))
tA = tA_all[None, (None, (tidx, None))]
a_regs = cute.make_rmem_tensor(
cute.make_layout((num_rows, vector_width), stride=(vector_width, 1)),
self.element_type,
)
b_regs = cute.make_rmem_tensor(
cute.make_layout(
(outputs_per_block, vector_width), stride=(vector_width, 1)
),
self.element_type,
)
for k_tile in cutlass.range(num_k_tiles, unroll=self.k_unroll):
for mi in cutlass.range_constexpr(num_rows):
cute.copy(copy_a, tA[mi, None, k_tile], a_regs[mi, None])
for ni in cutlass.range_constexpr(outputs_per_block):
n_idx = n_base + ni
tB = tB_all[n_idx, (None, (tidx, None))]
cute.copy(copy_b, tB[None, k_tile], b_regs[ni, None])
for vi in cutlass.range_constexpr(vector_width):
for mi in cutlass.range_constexpr(num_rows):
for ni in cutlass.range_constexpr(outputs_per_block):
acc[mi, ni] = acc[mi, ni] + a_regs[mi, vi].to(
cutlass.Float32
) * b_regs[ni, vi].to(cutlass.Float32)
for mi in cutlass.range_constexpr(num_rows):
for ni in cutlass.range_constexpr(outputs_per_block):
acc[mi, ni] = cute.arch.warp_reduction_sum(acc[mi, ni])
smem_layout = cute.make_layout(
(num_rows, outputs_per_block, num_warps),
stride=(outputs_per_block * num_warps, num_warps, 1),
)
smem = cutlass.utils.SmemAllocator()
partials = smem.allocate_tensor(cutlass.Float32, smem_layout, byte_alignment=16)
with cute.arch.elect_one():
for mi in cutlass.range_constexpr(num_rows):
for ni in cutlass.range_constexpr(outputs_per_block):
partials[mi, ni, warp_idx] = acc[mi, ni]
cute.arch.sync_threads()
if tidx == 0:
for mi in cutlass.range_constexpr(num_rows):
for ni in cutlass.range_constexpr(outputs_per_block):
n_idx = n_base + ni
total = (
partials[mi, ni, None]
.load()
.reduce(
cute.ReductionOp.ADD,
init_val=cutlass.Float32(0.0),
reduction_profile=0,
)
)
if const_expr(self.has_residual):
total += gResidual[mi, n_idx].to(cutlass.Float32)
gC[mi, n_idx] = cutlass.Float32(total).to(self.element_type)
if const_expr(self.use_pdl):
cute.arch.griddepcontrol_launch_dependents()
@@ -0,0 +1,252 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from __future__ import annotations
import logging
from collections.abc import Iterable
from dataclasses import dataclass
from functools import partial
from typing import Any
import torch
logger = logging.getLogger(__name__)
_cutedsl_available: bool | None = None
@dataclass(frozen=True, slots=True)
class SkinnyGemmConfig:
num_rows: int
block_size: int
outputs_per_block: int
k_unroll: int = 1
vector_width: int = 8
class ShapeDynamicSkinnyGemm:
def __init__(self) -> None:
self._compiled: dict[tuple[torch.dtype, SkinnyGemmConfig, bool], Any] = {}
self._warmup_configs: set[tuple[torch.dtype, SkinnyGemmConfig, bool]] = set()
self._warmup_registered = False
@staticmethod
def is_available() -> bool:
global _cutedsl_available
if _cutedsl_available is not None:
return _cutedsl_available
try:
import cutlass # noqa: F401
import cutlass.cute # noqa: F401
_cutedsl_available = True
except ImportError:
_cutedsl_available = False
logger.info("cuteDSL is not available; skinny GEMM is disabled")
return _cutedsl_available
@staticmethod
def _config(m: int, n: int, k: int) -> SkinnyGemmConfig:
num_rows = m
wide_block = 224
if m == 1 and k >= 7168 and k % (wide_block * 8) == 0:
if n % 3 == 0:
k_unroll = 2 if n <= 2304 else 4
return SkinnyGemmConfig(num_rows, wide_block, 3, k_unroll)
if 2304 < n < 4096 and n % 2 == 0:
return SkinnyGemmConfig(num_rows, wide_block, 2, k_unroll=4)
if k <= 2048 or k % (128 * 8) != 0:
outputs_per_block = 2 if k <= 2048 else 4
if n % outputs_per_block:
outputs_per_block = 1
if k % (64 * 8) == 0:
return SkinnyGemmConfig(num_rows, 64, outputs_per_block, 2)
if k % (32 * 8) == 0:
return SkinnyGemmConfig(num_rows, 32, outputs_per_block, 2)
return SkinnyGemmConfig(num_rows, 32, outputs_per_block, 2, vector_width=4)
block_size = 64 if 4096 <= n < 8192 else 128
outputs_per_block = 1 if m == 1 and n <= 2304 else 2
if n % outputs_per_block:
outputs_per_block = 1
k_unroll = 2 if n <= 2304 or n >= 16384 else 1
return SkinnyGemmConfig(
num_rows,
block_size,
outputs_per_block,
k_unroll=k_unroll,
)
@staticmethod
def _cutlass_dtype(dtype: torch.dtype):
from cutlass import BFloat16, Float16
return BFloat16 if dtype == torch.bfloat16 else Float16
@staticmethod
def _stream():
from cuda.bindings.driver import CUstream
from vllm.utils.torch_utils import current_stream
return CUstream(current_stream().cuda_stream)
@staticmethod
def _use_pdl() -> bool:
from vllm.platforms import current_platform
return current_platform.is_arch_support_pdl()
def _compile(
self,
dtype: torch.dtype,
config: SkinnyGemmConfig,
has_residual: bool,
) -> None:
import cutlass.cute as cute
from quack.compile_utils import make_fake_tensor
from ._skinny_gemm import CuteSkinnyGemm
element_type = self._cutlass_dtype(dtype)
n = cute.sym_int(divisibility=config.outputs_per_block)
k = cute.sym_int(divisibility=config.block_size * config.vector_width)
a = make_fake_tensor(
element_type,
(config.num_rows, k),
divisibility=config.vector_width,
)
b = make_fake_tensor(
element_type,
(n, k),
divisibility=config.vector_width,
)
c = make_fake_tensor(element_type, (config.num_rows, n), divisibility=1)
residual = make_fake_tensor(element_type, (config.num_rows, n), divisibility=1)
kernel = CuteSkinnyGemm(
element_type=element_type,
num_rows=config.num_rows,
block_size=config.block_size,
outputs_per_block=config.outputs_per_block,
vector_width=config.vector_width,
k_unroll=config.k_unroll,
has_residual=has_residual,
use_pdl=self._use_pdl(),
)
self._compiled[(dtype, config, has_residual)] = cute.compile(
kernel,
a,
b,
residual,
c,
self._stream(),
options="--enable-tvm-ffi --ptxas-options -maxrregcount=64",
)
def request_warmup(
self,
dtype: torch.dtype,
shapes: Iterable[tuple[int, int, int]],
) -> None:
"""Request compilation of ``(M, N, K)`` shapes before graph capture."""
self.request_warmup_configs(
dtype,
(self._config(m, n, k) for m, n, k in shapes),
)
def request_warmup_configs(
self,
dtype: torch.dtype,
configs: Iterable[SkinnyGemmConfig],
*,
has_residual: bool = False,
) -> None:
"""Request compilation of explicit measured configs before capture."""
self._warmup_configs.update((dtype, config, has_residual) for config in configs)
if self._warmup_registered:
return
from vllm.model_executor.warmup.cutedsl_warmup import (
register_cutedsl_warmup_provider,
)
register_cutedsl_warmup_provider(self)
self._warmup_registered = True
def get_cutedsl_warmup_compile_units(self):
from vllm.model_executor.warmup.cutedsl_warmup import CuTeDSLCompileUnit
return tuple(
CuTeDSLCompileUnit(
name=(
"shape-dynamic skinny GEMM with residual"
if has_residual
else "shape-dynamic skinny GEMM"
),
key=("shape-dynamic-skinny-gemm", dtype, config, has_residual),
compile=partial(self._compile, dtype, config, has_residual),
)
for dtype, config, has_residual in sorted(
self._warmup_configs,
key=lambda item: (
str(item[0]),
item[1].num_rows,
item[1].block_size,
item[1].outputs_per_block,
item[1].k_unroll,
item[1].vector_width,
item[2],
),
)
)
def __call__(
self,
a: torch.Tensor,
b: torch.Tensor,
config: SkinnyGemmConfig | None = None,
residual: torch.Tensor | None = None,
) -> torch.Tensor:
if a.dim() != 2 or b.dim() != 2:
raise ValueError("a and b must be 2D tensors")
if a.dtype not in (torch.bfloat16, torch.float16) or b.dtype != a.dtype:
raise ValueError("a and b must have the same BF16 or FP16 dtype")
if not a.is_cuda or not b.is_cuda or a.device != b.device:
raise ValueError("a and b must be CUDA tensors on the same device")
if not a.is_contiguous() or not b.is_contiguous():
raise ValueError("a and b must be contiguous")
if a.shape[1] != b.shape[1]:
raise ValueError("a and b must have matching K dimensions")
if not 1 <= a.shape[0] <= 16:
raise ValueError("shape-dynamic skinny GEMM requires 1 <= M <= 16")
if residual is not None:
if residual.dim() != 2 or residual.shape != (a.shape[0], b.shape[0]):
raise ValueError("residual must have shape (M, N)")
if residual.dtype != a.dtype:
raise ValueError("residual must have the same dtype as a and b")
if residual.device != a.device or not residual.is_cuda:
raise ValueError("residual must be on the same CUDA device")
if not residual.is_contiguous():
raise ValueError("residual must be contiguous")
config = config or self._config(a.shape[0], b.shape[0], a.shape[1])
if config.num_rows != a.shape[0]:
raise ValueError("config num_rows must match M")
if b.shape[0] % config.outputs_per_block != 0:
raise ValueError("N must be divisible by outputs_per_block")
if a.shape[1] % (config.block_size * config.vector_width) != 0:
raise ValueError(
"K must be divisible by block_size * vector_width for this config"
)
has_residual = residual is not None
cache_key = (a.dtype, config, has_residual)
if cache_key not in self._compiled:
self._compile(a.dtype, config, has_residual)
output = torch.empty((a.shape[0], b.shape[0]), dtype=a.dtype, device=a.device)
residual_arg = output if residual is None else residual
self._compiled[cache_key](a, b, residual_arg, output, self._stream())
return output
shape_dynamic_skinny_gemm = ShapeDynamicSkinnyGemm()