[Model] Add Kimi K3 support: model files and kernels [1/N] (#50089)
Signed-off-by: zjy0516 <riverclouds.zhu@qq.com> Signed-off-by: Isotr0py <Isotr0py@outlook.com> Signed-off-by: Bugen Zhao <i@bugenzhao.com> Signed-off-by: Yongye Zhu <zyy1102000@gmail.com> Co-authored-by: Isotr0py <Isotr0py@outlook.com> Co-authored-by: Thien Tran <gau.nernst@yahoo.com.sg> Co-authored-by: Bugen Zhao <i@bugenzhao.com> Co-authored-by: Yongye Zhu <zyy1102000@gmail.com> Co-authored-by: Ziming Huang <zelda.huanghuang@gmail.com> Co-authored-by: Roger Wang <hey@rogerw.io> Co-authored-by: Isotr0py <mozf@inferact.ai> Co-authored-by: aoshen02 <aoshen02@users.noreply.github.com> Co-authored-by: Woosuk Kwon <woosuk@inferact.ai> Co-authored-by: Jee Jee Li <jeejeelee@inferact.ai> Co-authored-by: aoshen02 <aoshen@inferact.ai> Co-authored-by: Summer Yang <girasoleyang@gmail.com> Co-authored-by: Kevin H. Luu <khluu000@gmail.com> Co-authored-by: Bowen Wang <abmfy@icloud.com> Co-authored-by: gnovack <novackgm@gmail.com> Co-authored-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai> Co-authored-by: xiaozhoupy <peiyuanzhou1994@gmail.com> Co-authored-by: Roy Wang <yasong.wang@inferact.ai> Co-authored-by: Jeff (Junze) Ma <93145857+majunze2001@users.noreply.github.com> Co-authored-by: Codex <codex@openai.com>
This commit is contained in:
co-authored by
Isotr0py
Thien Tran
Bugen Zhao
Yongye Zhu
Ziming Huang
Roger Wang
Isotr0py
aoshen02
Woosuk Kwon
Jee Jee Li
aoshen02
Summer Yang
Kevin H. Luu
Bowen Wang
gnovack
Nick Hill
Yifan Qiao
xiaozhoupy
Roy Wang
Jeff Ma
Codex
parent
6f00a1ae3b
commit
7c6729b769
+131
-8
@@ -2733,6 +2733,52 @@ def fused_minimax_m3_qknorm_rope_kv_insert(
|
||||
)
|
||||
|
||||
|
||||
def fused_kda_decode(
|
||||
x: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
bias: torch.Tensor | None,
|
||||
conv_state: torch.Tensor,
|
||||
raw_g: torch.Tensor,
|
||||
raw_beta: torch.Tensor,
|
||||
A_log: torch.Tensor,
|
||||
dt_bias: torch.Tensor,
|
||||
state_indices: torch.Tensor,
|
||||
state: torch.Tensor,
|
||||
out: torch.Tensor | None = None,
|
||||
lower_bound: float | None = None,
|
||||
output_gate: torch.Tensor | None = None,
|
||||
norm_weight: torch.Tensor | None = None,
|
||||
norm_eps: float = 1e-5,
|
||||
) -> torch.Tensor:
|
||||
if out is None:
|
||||
out = torch.empty(
|
||||
1,
|
||||
x.shape[0],
|
||||
raw_g.shape[2],
|
||||
raw_g.shape[3],
|
||||
dtype=x.dtype,
|
||||
device=x.device,
|
||||
)
|
||||
torch.ops._C.fused_kda_decode(
|
||||
x,
|
||||
weight,
|
||||
bias,
|
||||
conv_state,
|
||||
raw_g,
|
||||
raw_beta,
|
||||
A_log,
|
||||
dt_bias,
|
||||
state_indices,
|
||||
state,
|
||||
out,
|
||||
lower_bound,
|
||||
output_gate,
|
||||
norm_weight,
|
||||
norm_eps,
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
def concat_and_cache_mla(
|
||||
kv_c: torch.Tensor,
|
||||
k_pe: torch.Tensor,
|
||||
@@ -2746,6 +2792,26 @@ def concat_and_cache_mla(
|
||||
)
|
||||
|
||||
|
||||
def concat_and_cache_mla_grouped(
|
||||
kv_c: torch.Tensor,
|
||||
k_pe: torch.Tensor,
|
||||
kv_cache_ptrs: torch.Tensor,
|
||||
slot_mapping: torch.Tensor,
|
||||
block_size: int,
|
||||
block_stride: int,
|
||||
entry_stride: int,
|
||||
) -> None:
|
||||
torch.ops._C_cache_ops.concat_and_cache_mla_grouped(
|
||||
kv_c,
|
||||
k_pe,
|
||||
kv_cache_ptrs,
|
||||
slot_mapping,
|
||||
block_size,
|
||||
block_stride,
|
||||
entry_stride,
|
||||
)
|
||||
|
||||
|
||||
def kimi_k3_attn_res(
|
||||
prefix: torch.Tensor,
|
||||
delta: torch.Tensor,
|
||||
@@ -3040,6 +3106,63 @@ def all_reduce(
|
||||
torch.ops._C_custom_ar.all_reduce(fa, inp, out, reg_buffer, reg_buffer_sz_bytes)
|
||||
|
||||
|
||||
def custom_all_gather(
|
||||
fa: int,
|
||||
inp: torch.Tensor,
|
||||
out: torch.Tensor,
|
||||
reg_buffer: int,
|
||||
reg_buffer_sz_bytes: int,
|
||||
) -> None:
|
||||
torch.ops._C_custom_ar.custom_all_gather(
|
||||
fa, inp, out, reg_buffer, reg_buffer_sz_bytes
|
||||
)
|
||||
|
||||
|
||||
def mnnvl_lamport_all_gather(
|
||||
fa: int,
|
||||
inp: torch.Tensor,
|
||||
out: torch.Tensor,
|
||||
local_buffer: int,
|
||||
multicast_buffer: int,
|
||||
epoch_buffer: int,
|
||||
stage_sz_bytes: int,
|
||||
) -> None:
|
||||
torch.ops._C_custom_ar.mnnvl_lamport_all_gather(
|
||||
fa,
|
||||
inp,
|
||||
out,
|
||||
local_buffer,
|
||||
multicast_buffer,
|
||||
epoch_buffer,
|
||||
stage_sz_bytes,
|
||||
)
|
||||
|
||||
|
||||
def custom_reduce_scatter(
|
||||
fa: int,
|
||||
inp: torch.Tensor,
|
||||
out: torch.Tensor,
|
||||
reg_buffer: int,
|
||||
reg_buffer_sz_bytes: int,
|
||||
) -> None:
|
||||
torch.ops._C_custom_ar.custom_reduce_scatter(
|
||||
fa, inp, out, reg_buffer, reg_buffer_sz_bytes
|
||||
)
|
||||
|
||||
|
||||
def mnnvl_lamport_reduce_scatter(
|
||||
fa: int,
|
||||
inp: torch.Tensor,
|
||||
out: torch.Tensor,
|
||||
local_buffer: int,
|
||||
epoch_buffer: int,
|
||||
stage_sz_bytes: int,
|
||||
) -> None:
|
||||
torch.ops._C_custom_ar.mnnvl_lamport_reduce_scatter(
|
||||
fa, inp, out, local_buffer, epoch_buffer, stage_sz_bytes
|
||||
)
|
||||
|
||||
|
||||
def dispose(fa: int) -> None:
|
||||
torch.ops._C_custom_ar.dispose(fa)
|
||||
|
||||
@@ -3144,18 +3267,18 @@ def dsv3_fused_a_gemm(
|
||||
output: torch.Tensor,
|
||||
mat_a: torch.Tensor,
|
||||
mat_b: torch.Tensor,
|
||||
enable_pdl: bool = False,
|
||||
) -> None:
|
||||
"""DeepSeek V3 fused A GEMM (SM 9.0+, bf16 only, 1-16 tokens).
|
||||
"""Low-latency fused-A-style GEMM (SM 9.0+, BF16, 1-16 tokens).
|
||||
|
||||
Computes output = mat_a @ mat_b.T where:
|
||||
mat_a: [num_tokens, 7168] row-major bf16 (hidden states)
|
||||
mat_b: [7168, 2112] column-major bf16 (weight transposed)
|
||||
output: [num_tokens, 2112] row-major bf16
|
||||
Computes ``output = mat_a @ mat_b`` for the compiled Kimi K3 and
|
||||
DeepSeek V3 projection shapes. ``mat_a`` and ``output`` are row-major;
|
||||
``mat_b`` is the column-major transposed weight. ``enable_pdl`` permits
|
||||
programmatic dependent launch for callers that have validated it.
|
||||
|
||||
Optimized for the DeepSeek V2/V3 QKV A-projection at small batch sizes.
|
||||
Requires SM 9.0+ (Hopper).
|
||||
Requires SM 9.0+.
|
||||
"""
|
||||
torch.ops._C.dsv3_fused_a_gemm(output, mat_a, mat_b)
|
||||
torch.ops._C.dsv3_fused_a_gemm(output, mat_a, mat_b, enable_pdl)
|
||||
|
||||
|
||||
if hasattr(torch.ops._C, "weight_packed_linear"):
|
||||
|
||||
@@ -269,6 +269,7 @@ if TYPE_CHECKING:
|
||||
VLLM_NCCL_INCLUDE_PATH: str | None = None
|
||||
VLLM_GC_DEBUG: str = ""
|
||||
VLLM_DEBUG_WORKSPACE: bool = False
|
||||
VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION: bool = False
|
||||
VLLM_DISABLE_SHARED_EXPERTS_STREAM: bool = False
|
||||
VLLM_SHARED_EXPERTS_STREAM_TOKEN_THRESHOLD: int = 256
|
||||
VLLM_MULTI_STREAM_GEMM_TOKEN_THRESHOLD: int = 1024
|
||||
@@ -1894,6 +1895,11 @@ environment_variables: dict[str, Callable[[], Any]] = {
|
||||
# Debug workspace allocations.
|
||||
# logging of workspace resize operations.
|
||||
"VLLM_DEBUG_WORKSPACE": lambda: bool(int(os.getenv("VLLM_DEBUG_WORKSPACE", "0"))),
|
||||
# Enable the experimental Kimi K3 latent-MoE tail fusion.
|
||||
# Currently supported only on SM100 with TP=8/16 and BF16.
|
||||
"VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION": lambda: bool(
|
||||
int(os.getenv("VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION", "0"))
|
||||
),
|
||||
# Disables parallel execution of shared_experts via separate cuda stream
|
||||
"VLLM_DISABLE_SHARED_EXPERTS_STREAM": lambda: bool(
|
||||
int(os.getenv("VLLM_DISABLE_SHARED_EXPERTS_STREAM", "0"))
|
||||
|
||||
@@ -0,0 +1,180 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import cutlass
|
||||
import cutlass.cute as cute
|
||||
from cuda.bindings.driver import CUstream
|
||||
from cutlass import const_expr
|
||||
|
||||
|
||||
class CuteSkinnyGemm:
|
||||
"""Shape-dynamic low-latency GEMM for small token counts.
|
||||
|
||||
Computes ``C[M, N] = A[M, K] @ B[N, K].T + residual`` with BF16 or FP16
|
||||
inputs, FP32 accumulators, and an output matching the input dtype. The
|
||||
residual term is optional and is added before the output conversion. N and
|
||||
K are runtime values. The tiny M dimension is fully unrolled alongside a
|
||||
small set of tuning parameters.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
element_type,
|
||||
num_rows: int,
|
||||
block_size: int,
|
||||
outputs_per_block: int,
|
||||
vector_width: int = 8,
|
||||
k_unroll: int = 1,
|
||||
has_residual: bool = False,
|
||||
use_pdl: bool = False,
|
||||
) -> None:
|
||||
if block_size % cute.arch.WARP_SIZE != 0:
|
||||
raise ValueError("block_size must be a multiple of the warp size")
|
||||
self.element_type = element_type
|
||||
self.num_rows = num_rows
|
||||
self.block_size = block_size
|
||||
self.outputs_per_block = outputs_per_block
|
||||
self.vector_width = vector_width
|
||||
self.k_unroll = k_unroll
|
||||
self.has_residual = has_residual
|
||||
self.use_pdl = use_pdl
|
||||
self.num_warps = block_size // cute.arch.WARP_SIZE
|
||||
|
||||
@cute.jit
|
||||
def __call__(
|
||||
self,
|
||||
gA: cute.Tensor,
|
||||
gB: cute.Tensor,
|
||||
gResidual: cute.Tensor,
|
||||
gC: cute.Tensor,
|
||||
stream: CUstream,
|
||||
) -> None:
|
||||
n = cute.size(gB, mode=[0])
|
||||
k = cute.size(gA, mode=[1])
|
||||
copy_a = cute.make_copy_atom(
|
||||
cute.nvgpu.CopyG2ROp(),
|
||||
self.element_type,
|
||||
num_bits_per_copy=self.vector_width * self.element_type.width,
|
||||
load_cache_mode=cute.nvgpu.LoadCacheMode.ALWAYS,
|
||||
)
|
||||
copy_b = cute.make_copy_atom(
|
||||
cute.nvgpu.CopyG2ROp(),
|
||||
self.element_type,
|
||||
num_bits_per_copy=self.vector_width * self.element_type.width,
|
||||
load_cache_mode=cute.nvgpu.LoadCacheMode.STREAMING,
|
||||
)
|
||||
self.kernel(gA, gB, gResidual, gC, k, copy_a, copy_b).launch(
|
||||
grid=[cute.ceil_div(n, self.outputs_per_block), 1, 1],
|
||||
block=[self.block_size, 1, 1],
|
||||
smem=self.num_rows * self.outputs_per_block * self.num_warps * 4,
|
||||
stream=stream,
|
||||
use_pdl=self.use_pdl,
|
||||
min_blocks_per_mp=1,
|
||||
)
|
||||
|
||||
@cute.kernel
|
||||
def kernel(
|
||||
self,
|
||||
gA: cute.Tensor,
|
||||
gB: cute.Tensor,
|
||||
gResidual: cute.Tensor,
|
||||
gC: cute.Tensor,
|
||||
k_extent: cutlass.Int32,
|
||||
copy_a: cute.CopyAtom,
|
||||
copy_b: cute.CopyAtom,
|
||||
) -> None:
|
||||
tidx, _, _ = cute.arch.thread_idx()
|
||||
block_idx, _, _ = cute.arch.block_idx()
|
||||
warp_idx = cute.arch.warp_idx()
|
||||
|
||||
num_rows: cutlass.Constexpr = self.num_rows
|
||||
outputs_per_block: cutlass.Constexpr = self.outputs_per_block
|
||||
vector_width: cutlass.Constexpr = self.vector_width
|
||||
block_size: cutlass.Constexpr = self.block_size
|
||||
num_warps: cutlass.Constexpr = self.num_warps
|
||||
|
||||
acc_layout = cute.make_layout(
|
||||
(num_rows, outputs_per_block), stride=(outputs_per_block, 1)
|
||||
)
|
||||
acc = cute.make_rmem_tensor(acc_layout, cutlass.Float32)
|
||||
acc.fill(0.0)
|
||||
|
||||
if const_expr(self.use_pdl):
|
||||
cute.arch.griddepcontrol_wait()
|
||||
|
||||
n_base = block_idx * outputs_per_block
|
||||
k_tile_size: cutlass.Constexpr = block_size * vector_width
|
||||
num_k_tiles = k_extent // k_tile_size
|
||||
|
||||
gA_vec = cute.logical_divide(gA, (None, vector_width))
|
||||
gB_vec = cute.logical_divide(gB, (None, vector_width))
|
||||
# Layout after both divides is (M/N, K_TILE, K_LANE, K_VEC).
|
||||
tA_all = cute.logical_divide(gA_vec, (None, (None, block_size)))
|
||||
tB_all = cute.logical_divide(gB_vec, (None, (None, block_size)))
|
||||
tA = tA_all[None, (None, (tidx, None))]
|
||||
|
||||
a_regs = cute.make_rmem_tensor(
|
||||
cute.make_layout((num_rows, vector_width), stride=(vector_width, 1)),
|
||||
self.element_type,
|
||||
)
|
||||
b_regs = cute.make_rmem_tensor(
|
||||
cute.make_layout(
|
||||
(outputs_per_block, vector_width), stride=(vector_width, 1)
|
||||
),
|
||||
self.element_type,
|
||||
)
|
||||
|
||||
for k_tile in cutlass.range(num_k_tiles, unroll=self.k_unroll):
|
||||
for mi in cutlass.range_constexpr(num_rows):
|
||||
cute.copy(copy_a, tA[mi, None, k_tile], a_regs[mi, None])
|
||||
|
||||
for ni in cutlass.range_constexpr(outputs_per_block):
|
||||
n_idx = n_base + ni
|
||||
tB = tB_all[n_idx, (None, (tidx, None))]
|
||||
cute.copy(copy_b, tB[None, k_tile], b_regs[ni, None])
|
||||
|
||||
for vi in cutlass.range_constexpr(vector_width):
|
||||
for mi in cutlass.range_constexpr(num_rows):
|
||||
for ni in cutlass.range_constexpr(outputs_per_block):
|
||||
acc[mi, ni] = acc[mi, ni] + a_regs[mi, vi].to(
|
||||
cutlass.Float32
|
||||
) * b_regs[ni, vi].to(cutlass.Float32)
|
||||
|
||||
for mi in cutlass.range_constexpr(num_rows):
|
||||
for ni in cutlass.range_constexpr(outputs_per_block):
|
||||
acc[mi, ni] = cute.arch.warp_reduction_sum(acc[mi, ni])
|
||||
|
||||
smem_layout = cute.make_layout(
|
||||
(num_rows, outputs_per_block, num_warps),
|
||||
stride=(outputs_per_block * num_warps, num_warps, 1),
|
||||
)
|
||||
smem = cutlass.utils.SmemAllocator()
|
||||
partials = smem.allocate_tensor(cutlass.Float32, smem_layout, byte_alignment=16)
|
||||
with cute.arch.elect_one():
|
||||
for mi in cutlass.range_constexpr(num_rows):
|
||||
for ni in cutlass.range_constexpr(outputs_per_block):
|
||||
partials[mi, ni, warp_idx] = acc[mi, ni]
|
||||
|
||||
cute.arch.sync_threads()
|
||||
if tidx == 0:
|
||||
for mi in cutlass.range_constexpr(num_rows):
|
||||
for ni in cutlass.range_constexpr(outputs_per_block):
|
||||
n_idx = n_base + ni
|
||||
total = (
|
||||
partials[mi, ni, None]
|
||||
.load()
|
||||
.reduce(
|
||||
cute.ReductionOp.ADD,
|
||||
init_val=cutlass.Float32(0.0),
|
||||
reduction_profile=0,
|
||||
)
|
||||
)
|
||||
if const_expr(self.has_residual):
|
||||
total += gResidual[mi, n_idx].to(cutlass.Float32)
|
||||
gC[mi, n_idx] = cutlass.Float32(total).to(self.element_type)
|
||||
|
||||
if const_expr(self.use_pdl):
|
||||
cute.arch.griddepcontrol_launch_dependents()
|
||||
@@ -0,0 +1,252 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from collections.abc import Iterable
|
||||
from dataclasses import dataclass
|
||||
from functools import partial
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_cutedsl_available: bool | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class SkinnyGemmConfig:
|
||||
num_rows: int
|
||||
block_size: int
|
||||
outputs_per_block: int
|
||||
k_unroll: int = 1
|
||||
vector_width: int = 8
|
||||
|
||||
|
||||
class ShapeDynamicSkinnyGemm:
|
||||
def __init__(self) -> None:
|
||||
self._compiled: dict[tuple[torch.dtype, SkinnyGemmConfig, bool], Any] = {}
|
||||
self._warmup_configs: set[tuple[torch.dtype, SkinnyGemmConfig, bool]] = set()
|
||||
self._warmup_registered = False
|
||||
|
||||
@staticmethod
|
||||
def is_available() -> bool:
|
||||
global _cutedsl_available
|
||||
if _cutedsl_available is not None:
|
||||
return _cutedsl_available
|
||||
try:
|
||||
import cutlass # noqa: F401
|
||||
import cutlass.cute # noqa: F401
|
||||
|
||||
_cutedsl_available = True
|
||||
except ImportError:
|
||||
_cutedsl_available = False
|
||||
logger.info("cuteDSL is not available; skinny GEMM is disabled")
|
||||
return _cutedsl_available
|
||||
|
||||
@staticmethod
|
||||
def _config(m: int, n: int, k: int) -> SkinnyGemmConfig:
|
||||
num_rows = m
|
||||
wide_block = 224
|
||||
if m == 1 and k >= 7168 and k % (wide_block * 8) == 0:
|
||||
if n % 3 == 0:
|
||||
k_unroll = 2 if n <= 2304 else 4
|
||||
return SkinnyGemmConfig(num_rows, wide_block, 3, k_unroll)
|
||||
if 2304 < n < 4096 and n % 2 == 0:
|
||||
return SkinnyGemmConfig(num_rows, wide_block, 2, k_unroll=4)
|
||||
|
||||
if k <= 2048 or k % (128 * 8) != 0:
|
||||
outputs_per_block = 2 if k <= 2048 else 4
|
||||
if n % outputs_per_block:
|
||||
outputs_per_block = 1
|
||||
if k % (64 * 8) == 0:
|
||||
return SkinnyGemmConfig(num_rows, 64, outputs_per_block, 2)
|
||||
if k % (32 * 8) == 0:
|
||||
return SkinnyGemmConfig(num_rows, 32, outputs_per_block, 2)
|
||||
return SkinnyGemmConfig(num_rows, 32, outputs_per_block, 2, vector_width=4)
|
||||
|
||||
block_size = 64 if 4096 <= n < 8192 else 128
|
||||
outputs_per_block = 1 if m == 1 and n <= 2304 else 2
|
||||
if n % outputs_per_block:
|
||||
outputs_per_block = 1
|
||||
k_unroll = 2 if n <= 2304 or n >= 16384 else 1
|
||||
return SkinnyGemmConfig(
|
||||
num_rows,
|
||||
block_size,
|
||||
outputs_per_block,
|
||||
k_unroll=k_unroll,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _cutlass_dtype(dtype: torch.dtype):
|
||||
from cutlass import BFloat16, Float16
|
||||
|
||||
return BFloat16 if dtype == torch.bfloat16 else Float16
|
||||
|
||||
@staticmethod
|
||||
def _stream():
|
||||
from cuda.bindings.driver import CUstream
|
||||
|
||||
from vllm.utils.torch_utils import current_stream
|
||||
|
||||
return CUstream(current_stream().cuda_stream)
|
||||
|
||||
@staticmethod
|
||||
def _use_pdl() -> bool:
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
return current_platform.is_arch_support_pdl()
|
||||
|
||||
def _compile(
|
||||
self,
|
||||
dtype: torch.dtype,
|
||||
config: SkinnyGemmConfig,
|
||||
has_residual: bool,
|
||||
) -> None:
|
||||
import cutlass.cute as cute
|
||||
from quack.compile_utils import make_fake_tensor
|
||||
|
||||
from ._skinny_gemm import CuteSkinnyGemm
|
||||
|
||||
element_type = self._cutlass_dtype(dtype)
|
||||
n = cute.sym_int(divisibility=config.outputs_per_block)
|
||||
k = cute.sym_int(divisibility=config.block_size * config.vector_width)
|
||||
a = make_fake_tensor(
|
||||
element_type,
|
||||
(config.num_rows, k),
|
||||
divisibility=config.vector_width,
|
||||
)
|
||||
b = make_fake_tensor(
|
||||
element_type,
|
||||
(n, k),
|
||||
divisibility=config.vector_width,
|
||||
)
|
||||
c = make_fake_tensor(element_type, (config.num_rows, n), divisibility=1)
|
||||
residual = make_fake_tensor(element_type, (config.num_rows, n), divisibility=1)
|
||||
kernel = CuteSkinnyGemm(
|
||||
element_type=element_type,
|
||||
num_rows=config.num_rows,
|
||||
block_size=config.block_size,
|
||||
outputs_per_block=config.outputs_per_block,
|
||||
vector_width=config.vector_width,
|
||||
k_unroll=config.k_unroll,
|
||||
has_residual=has_residual,
|
||||
use_pdl=self._use_pdl(),
|
||||
)
|
||||
self._compiled[(dtype, config, has_residual)] = cute.compile(
|
||||
kernel,
|
||||
a,
|
||||
b,
|
||||
residual,
|
||||
c,
|
||||
self._stream(),
|
||||
options="--enable-tvm-ffi --ptxas-options -maxrregcount=64",
|
||||
)
|
||||
|
||||
def request_warmup(
|
||||
self,
|
||||
dtype: torch.dtype,
|
||||
shapes: Iterable[tuple[int, int, int]],
|
||||
) -> None:
|
||||
"""Request compilation of ``(M, N, K)`` shapes before graph capture."""
|
||||
self.request_warmup_configs(
|
||||
dtype,
|
||||
(self._config(m, n, k) for m, n, k in shapes),
|
||||
)
|
||||
|
||||
def request_warmup_configs(
|
||||
self,
|
||||
dtype: torch.dtype,
|
||||
configs: Iterable[SkinnyGemmConfig],
|
||||
*,
|
||||
has_residual: bool = False,
|
||||
) -> None:
|
||||
"""Request compilation of explicit measured configs before capture."""
|
||||
self._warmup_configs.update((dtype, config, has_residual) for config in configs)
|
||||
if self._warmup_registered:
|
||||
return
|
||||
from vllm.model_executor.warmup.cutedsl_warmup import (
|
||||
register_cutedsl_warmup_provider,
|
||||
)
|
||||
|
||||
register_cutedsl_warmup_provider(self)
|
||||
self._warmup_registered = True
|
||||
|
||||
def get_cutedsl_warmup_compile_units(self):
|
||||
from vllm.model_executor.warmup.cutedsl_warmup import CuTeDSLCompileUnit
|
||||
|
||||
return tuple(
|
||||
CuTeDSLCompileUnit(
|
||||
name=(
|
||||
"shape-dynamic skinny GEMM with residual"
|
||||
if has_residual
|
||||
else "shape-dynamic skinny GEMM"
|
||||
),
|
||||
key=("shape-dynamic-skinny-gemm", dtype, config, has_residual),
|
||||
compile=partial(self._compile, dtype, config, has_residual),
|
||||
)
|
||||
for dtype, config, has_residual in sorted(
|
||||
self._warmup_configs,
|
||||
key=lambda item: (
|
||||
str(item[0]),
|
||||
item[1].num_rows,
|
||||
item[1].block_size,
|
||||
item[1].outputs_per_block,
|
||||
item[1].k_unroll,
|
||||
item[1].vector_width,
|
||||
item[2],
|
||||
),
|
||||
)
|
||||
)
|
||||
|
||||
def __call__(
|
||||
self,
|
||||
a: torch.Tensor,
|
||||
b: torch.Tensor,
|
||||
config: SkinnyGemmConfig | None = None,
|
||||
residual: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
if a.dim() != 2 or b.dim() != 2:
|
||||
raise ValueError("a and b must be 2D tensors")
|
||||
if a.dtype not in (torch.bfloat16, torch.float16) or b.dtype != a.dtype:
|
||||
raise ValueError("a and b must have the same BF16 or FP16 dtype")
|
||||
if not a.is_cuda or not b.is_cuda or a.device != b.device:
|
||||
raise ValueError("a and b must be CUDA tensors on the same device")
|
||||
if not a.is_contiguous() or not b.is_contiguous():
|
||||
raise ValueError("a and b must be contiguous")
|
||||
if a.shape[1] != b.shape[1]:
|
||||
raise ValueError("a and b must have matching K dimensions")
|
||||
if not 1 <= a.shape[0] <= 16:
|
||||
raise ValueError("shape-dynamic skinny GEMM requires 1 <= M <= 16")
|
||||
if residual is not None:
|
||||
if residual.dim() != 2 or residual.shape != (a.shape[0], b.shape[0]):
|
||||
raise ValueError("residual must have shape (M, N)")
|
||||
if residual.dtype != a.dtype:
|
||||
raise ValueError("residual must have the same dtype as a and b")
|
||||
if residual.device != a.device or not residual.is_cuda:
|
||||
raise ValueError("residual must be on the same CUDA device")
|
||||
if not residual.is_contiguous():
|
||||
raise ValueError("residual must be contiguous")
|
||||
|
||||
config = config or self._config(a.shape[0], b.shape[0], a.shape[1])
|
||||
if config.num_rows != a.shape[0]:
|
||||
raise ValueError("config num_rows must match M")
|
||||
if b.shape[0] % config.outputs_per_block != 0:
|
||||
raise ValueError("N must be divisible by outputs_per_block")
|
||||
if a.shape[1] % (config.block_size * config.vector_width) != 0:
|
||||
raise ValueError(
|
||||
"K must be divisible by block_size * vector_width for this config"
|
||||
)
|
||||
has_residual = residual is not None
|
||||
cache_key = (a.dtype, config, has_residual)
|
||||
if cache_key not in self._compiled:
|
||||
self._compile(a.dtype, config, has_residual)
|
||||
output = torch.empty((a.shape[0], b.shape[0]), dtype=a.dtype, device=a.device)
|
||||
residual_arg = output if residual is None else residual
|
||||
self._compiled[cache_key](a, b, residual_arg, output, self._stream())
|
||||
return output
|
||||
|
||||
|
||||
shape_dynamic_skinny_gemm = ShapeDynamicSkinnyGemm()
|
||||
@@ -158,6 +158,51 @@ class SiluAndMul(CustomOp):
|
||||
return self.forward_native(x)
|
||||
|
||||
|
||||
@CustomOp.register("situ_and_mul")
|
||||
class SituAndMul(CustomOp):
|
||||
"""SituGLU activation used by Kimi models.
|
||||
|
||||
Computes beta * tanh(gate / beta) * sigmoid(gate) * up. When
|
||||
``linear_beta`` is set, the up projection is also softly clipped with
|
||||
linear_beta * tanh(up / linear_beta).
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
beta: float = 1.0,
|
||||
linear_beta: float | None = None,
|
||||
*,
|
||||
compile_native: bool = True,
|
||||
):
|
||||
super().__init__(compile_native=compile_native)
|
||||
self.beta = float(beta)
|
||||
self.linear_beta = None if linear_beta is None else float(linear_beta)
|
||||
if current_platform.is_cuda_alike():
|
||||
self.op = torch.ops._C.situ_and_mul
|
||||
|
||||
def forward_native(self, x: torch.Tensor) -> torch.Tensor:
|
||||
d = x.shape[-1] // 2
|
||||
gate = x[..., :d].float()
|
||||
up = x[..., d:].float()
|
||||
gate = self.beta * torch.tanh(gate / self.beta) * torch.sigmoid(gate)
|
||||
if self.linear_beta is not None:
|
||||
up = self.linear_beta * torch.tanh(up / self.linear_beta)
|
||||
return (gate * up).to(x.dtype)
|
||||
|
||||
def forward_cuda(self, x: torch.Tensor) -> torch.Tensor:
|
||||
# Fused CUDA kernel: writes straight to `out`, no fp32 temporaries.
|
||||
# linear_beta<=0 signals "unset" to the kernel (up passed through).
|
||||
d = x.shape[-1] // 2
|
||||
out = torch.empty(x.shape[:-1] + (d,), dtype=x.dtype, device=x.device)
|
||||
self.op(
|
||||
out, x, self.beta, -1.0 if self.linear_beta is None else self.linear_beta
|
||||
)
|
||||
return out
|
||||
|
||||
def forward_xpu(self, x: torch.Tensor) -> torch.Tensor:
|
||||
return self.forward_native(x)
|
||||
|
||||
|
||||
@CustomOp.register("silu_and_mul_with_clamp")
|
||||
class SiluAndMulWithClamp(CustomOp):
|
||||
"""SwiGLU activation with input clamping (used by some MoE shared experts).
|
||||
|
||||
@@ -22,6 +22,7 @@ class MoEActivation(Enum):
|
||||
# expects the *packed* layout ([all gates; all ups]), as produced by a
|
||||
# MergedColumnParallelLinear gate_up_proj (e.g. MiniMax-M3).
|
||||
SWIGLUOAI = "swigluoai"
|
||||
SITU = "situ"
|
||||
SWIGLUOAI_UNINTERLEAVE = "swigluoai_uninterleave"
|
||||
SWIGLUSTEP = "swiglustep"
|
||||
|
||||
@@ -77,6 +78,7 @@ _CUSTOM_OP_NAMES: dict[MoEActivation, str] = {
|
||||
MoEActivation.SILU: "silu_and_mul",
|
||||
MoEActivation.GELU: "gelu_and_mul",
|
||||
MoEActivation.GELU_TANH: "gelu_tanh_and_mul",
|
||||
MoEActivation.SITU: "situ_and_mul",
|
||||
MoEActivation.SWIGLUOAI: "swigluoai_and_mul",
|
||||
MoEActivation.SWIGLUOAI_UNINTERLEAVE: "silu_and_mul_with_clamp",
|
||||
MoEActivation.SWIGLUSTEP: "swiglustep_and_mul",
|
||||
@@ -133,6 +135,8 @@ def apply_moe_activation(
|
||||
beta: float = 0.0,
|
||||
topk_ids: torch.Tensor | None = None,
|
||||
expert_map: torch.Tensor | None = None,
|
||||
activation_situ_beta: float | None = None,
|
||||
activation_situ_linear_beta: float | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""Apply MoE activation function.
|
||||
|
||||
@@ -163,6 +167,25 @@ def apply_moe_activation(
|
||||
torch.ops._C.gelu_and_mul(output, input)
|
||||
elif activation == MoEActivation.GELU_TANH:
|
||||
torch.ops._C.gelu_tanh_and_mul(output, input)
|
||||
elif activation == MoEActivation.SITU:
|
||||
# Fused CUDA kernel: writes straight to `output`, no fp32 temporaries.
|
||||
# (The pure-torch fallback below upcast both halves to fp32 and
|
||||
# allocated ~8 temporaries per call, blowing up MoE memory.)
|
||||
# Both betas come from FusedMoEConfig; a missing beta means the caller
|
||||
# bypassed the config plumbing, so fail rather than silently use 1.0.
|
||||
# linear_beta is genuinely optional: <= 0 signals "unset" to the kernel
|
||||
# (up passed through), matching SituAndMul(linear_beta=None).
|
||||
assert activation_situ_beta is not None, (
|
||||
"SITU requires activation_situ_beta from FusedMoEConfig"
|
||||
)
|
||||
torch.ops._C.situ_and_mul(
|
||||
output,
|
||||
input,
|
||||
activation_situ_beta,
|
||||
-1.0
|
||||
if activation_situ_linear_beta is None
|
||||
else activation_situ_linear_beta,
|
||||
)
|
||||
elif activation == MoEActivation.SWIGLUOAI:
|
||||
torch.ops._C.swigluoai_and_mul(output, input)
|
||||
elif activation == MoEActivation.SWIGLUOAI_UNINTERLEAVE:
|
||||
|
||||
@@ -1314,6 +1314,10 @@ class FusedMoEConfig:
|
||||
swiglu_alpha: float | None = None
|
||||
swiglu_beta: float | None = None
|
||||
|
||||
# SituGLU parameters used by Kimi sit(u/v2) activations.
|
||||
activation_situ_beta: float | None = None
|
||||
activation_situ_linear_beta: float | None = None
|
||||
|
||||
max_capture_size: int = 0
|
||||
|
||||
# Set by __post_init__
|
||||
|
||||
@@ -444,7 +444,14 @@ class DeepGemmFP4Experts(mk.FusedMoEExpertsModular):
|
||||
|
||||
@staticmethod
|
||||
def _supports_activation(activation: MoEActivation) -> bool:
|
||||
return activation in [MoEActivation.SILU, MoEActivation.SWIGLUSTEP]
|
||||
# SILU has fused gate+mul+quant kernels; SWIGLUSTEP/SITU take the
|
||||
# general path (activation applied via self.activation, which forwards
|
||||
# the situ betas, then FP8 requant).
|
||||
return activation in [
|
||||
MoEActivation.SILU,
|
||||
MoEActivation.SWIGLUSTEP,
|
||||
MoEActivation.SITU,
|
||||
]
|
||||
|
||||
@staticmethod
|
||||
def _supports_parallel_config(moe_parallel_config: FusedMoEParallelConfig) -> bool:
|
||||
@@ -488,16 +495,15 @@ class DeepGemmFP4Experts(mk.FusedMoEExpertsModular):
|
||||
M_sum, N = input.size()
|
||||
activation_out_dim = self.adjust_N_for_activation(N, activation)
|
||||
|
||||
if scale_fmt == DeepGemmQuantScaleFMT.UE8M0:
|
||||
assert activation == MoEActivation.SILU
|
||||
return fused_silu_mul_fp8_quant_packed(
|
||||
input=input,
|
||||
output_q=output,
|
||||
group_size=block_k,
|
||||
clamp_limit=self.gemm1_clamp_limit,
|
||||
)
|
||||
|
||||
if activation == MoEActivation.SILU:
|
||||
# Fused gate+mul+quant kernels for the common SILU case.
|
||||
if scale_fmt == DeepGemmQuantScaleFMT.UE8M0:
|
||||
return fused_silu_mul_fp8_quant_packed(
|
||||
input=input,
|
||||
output_q=output,
|
||||
group_size=block_k,
|
||||
clamp_limit=self.gemm1_clamp_limit,
|
||||
)
|
||||
use_ue8m0 = scale_fmt == DeepGemmQuantScaleFMT.FLOAT32_CEIL_UE8M0
|
||||
return silu_mul_per_token_group_quant_fp8_colmajor(
|
||||
input=input,
|
||||
@@ -506,12 +512,24 @@ class DeepGemmFP4Experts(mk.FusedMoEExpertsModular):
|
||||
clamp_limit=self.gemm1_clamp_limit,
|
||||
)
|
||||
|
||||
# General gated activations (SWIGLUSTEP, SITU): apply the activation
|
||||
# (self.activation forwards the situ betas from moe_config) then
|
||||
# FP8-requant into the layout DeepGEMM expects for this scale format.
|
||||
act_out = torch.empty(
|
||||
(M_sum, activation_out_dim), dtype=input.dtype, device=input.device
|
||||
)
|
||||
self.activation(activation, act_out, input)
|
||||
if scale_fmt == DeepGemmQuantScaleFMT.UE8M0:
|
||||
return per_token_group_quant_fp8_packed_for_deepgemm(
|
||||
act_out, block_k, use_ue8m0=True, out_q=output
|
||||
)
|
||||
use_ue8m0 = scale_fmt == DeepGemmQuantScaleFMT.FLOAT32_CEIL_UE8M0
|
||||
return per_token_group_quant_fp8(
|
||||
act_out, block_k, column_major_scales=True, out_q=output
|
||||
act_out,
|
||||
block_k,
|
||||
column_major_scales=True,
|
||||
out_q=output,
|
||||
use_ue8m0=use_ue8m0,
|
||||
)
|
||||
|
||||
def apply(
|
||||
|
||||
@@ -93,6 +93,8 @@ def _fused_marlin_moe(
|
||||
clamp_limit: float | None = None,
|
||||
gemm1_alpha: float = 1.0,
|
||||
gemm1_beta: float = 0.0,
|
||||
activation_situ_beta: float | None = None,
|
||||
activation_situ_linear_beta: float | None = None,
|
||||
) -> torch.Tensor:
|
||||
assert hidden_states.ndim == 2
|
||||
M, K = hidden_states.size()
|
||||
@@ -171,6 +173,8 @@ def _fused_marlin_moe(
|
||||
beta=gemm1_beta,
|
||||
topk_ids=topk_ids,
|
||||
expert_map=expert_map,
|
||||
activation_situ_beta=activation_situ_beta,
|
||||
activation_situ_linear_beta=activation_situ_linear_beta,
|
||||
)
|
||||
|
||||
if output is None:
|
||||
@@ -256,6 +260,8 @@ def fused_marlin_moe(
|
||||
clamp_limit: float | None = None,
|
||||
gemm1_alpha: float = 1.0,
|
||||
gemm1_beta: float = 0.0,
|
||||
activation_situ_beta: float | None = None,
|
||||
activation_situ_linear_beta: float | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
This function computes a Mixture of Experts (MoE) layer using two sets of
|
||||
@@ -357,6 +363,8 @@ def fused_marlin_moe(
|
||||
num_tokens_post_padded=num_tokens_post_padded,
|
||||
activation=activation,
|
||||
activation_func=activation_func,
|
||||
activation_situ_beta=activation_situ_beta,
|
||||
activation_situ_linear_beta=activation_situ_linear_beta,
|
||||
input_global_scale1=input_global_scale1,
|
||||
input_global_scale2=input_global_scale2,
|
||||
global_scale1=global_scale1,
|
||||
@@ -423,6 +431,9 @@ def batched_fused_marlin_moe(
|
||||
clamp_limit: float | None = None,
|
||||
gemm1_alpha: float = 1.0,
|
||||
gemm1_beta: float = 0.0,
|
||||
activation_func: Callable[..., None] = apply_moe_activation,
|
||||
activation_situ_beta: float | None = None,
|
||||
activation_situ_linear_beta: float | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
This function massages the inputs so the batched hidden_states can be
|
||||
@@ -530,6 +541,9 @@ def batched_fused_marlin_moe(
|
||||
quant_type=quant_type,
|
||||
apply_router_weight_on_input=apply_router_weight_on_input,
|
||||
activation=activation,
|
||||
activation_func=activation_func,
|
||||
activation_situ_beta=activation_situ_beta,
|
||||
activation_situ_linear_beta=activation_situ_linear_beta,
|
||||
expert_map=expert_map,
|
||||
block_size_m=block_size_m,
|
||||
sorted_token_ids=sorted_token_ids,
|
||||
@@ -645,6 +659,7 @@ class MarlinExpertsBase(mk.FusedMoEExpertsModular):
|
||||
MoEActivation.SILU,
|
||||
MoEActivation.GELU,
|
||||
MoEActivation.GELU_TANH,
|
||||
MoEActivation.SITU,
|
||||
MoEActivation.SWIGLUOAI,
|
||||
MoEActivation.SWIGLUOAI_UNINTERLEAVE,
|
||||
MoEActivation.SWIGLUSTEP,
|
||||
@@ -793,6 +808,10 @@ class MarlinExperts(LoRAExpertsMixin, MarlinExpertsBase):
|
||||
global_num_experts=global_num_experts,
|
||||
activation=activation,
|
||||
activation_func=self.activation,
|
||||
activation_situ_beta=self.moe_config.activation_situ_beta,
|
||||
activation_situ_linear_beta=(
|
||||
self.moe_config.activation_situ_linear_beta
|
||||
),
|
||||
moe_sum=self.moe_sum,
|
||||
expert_map=expert_map,
|
||||
output=output,
|
||||
@@ -833,6 +852,8 @@ class MarlinExperts(LoRAExpertsMixin, MarlinExpertsBase):
|
||||
beta: float = 0.0,
|
||||
topk_ids: torch.Tensor | None = None,
|
||||
expert_map: torch.Tensor | None = None,
|
||||
activation_situ_beta: float | None = None,
|
||||
activation_situ_linear_beta: float | None = None,
|
||||
) -> None:
|
||||
# act_input = intermediate_cache1 (M*topk, 2N for gated)
|
||||
# act_output = intermediate_cache2 (M*topk, N)
|
||||
@@ -871,6 +892,8 @@ class MarlinExperts(LoRAExpertsMixin, MarlinExpertsBase):
|
||||
beta=beta,
|
||||
topk_ids=topk_ids,
|
||||
expert_map=expert_map,
|
||||
activation_situ_beta=activation_situ_beta,
|
||||
activation_situ_linear_beta=activation_situ_linear_beta,
|
||||
)
|
||||
lora_state["cache2"] = act_output
|
||||
|
||||
@@ -918,6 +941,8 @@ class MarlinExperts(LoRAExpertsMixin, MarlinExpertsBase):
|
||||
global_num_experts=global_num_experts,
|
||||
activation=activation,
|
||||
activation_func=activation_with_lora,
|
||||
activation_situ_beta=self.moe_config.activation_situ_beta,
|
||||
activation_situ_linear_beta=self.moe_config.activation_situ_linear_beta,
|
||||
moe_sum=moe_sum_with_lora,
|
||||
expert_map=expert_map,
|
||||
output=output,
|
||||
@@ -1021,6 +1046,41 @@ class BatchedMarlinExperts(MarlinExpertsBase):
|
||||
apply_router_weight_on_input: bool,
|
||||
):
|
||||
assert expert_tokens_meta is not None, "Num valid tokens per batch is required"
|
||||
|
||||
def activation_func(
|
||||
act: MoEActivation,
|
||||
act_output: torch.Tensor,
|
||||
act_input: torch.Tensor,
|
||||
*,
|
||||
activation_situ_beta: float | None = None,
|
||||
activation_situ_linear_beta: float | None = None,
|
||||
**kwargs,
|
||||
) -> None:
|
||||
if act != MoEActivation.SITU:
|
||||
self.activation(
|
||||
act,
|
||||
act_output,
|
||||
act_input,
|
||||
activation_situ_beta=activation_situ_beta,
|
||||
activation_situ_linear_beta=activation_situ_linear_beta,
|
||||
**kwargs,
|
||||
)
|
||||
return
|
||||
|
||||
num_experts, max_num_tokens = hidden_states.shape[:2]
|
||||
beta = activation_situ_beta
|
||||
linear_beta = activation_situ_linear_beta
|
||||
assert beta is not None, (
|
||||
"SITU requires activation_situ_beta from FusedMoEConfig"
|
||||
)
|
||||
torch.ops._C.masked_situ_and_mul(
|
||||
act_output.view(num_experts, max_num_tokens, -1),
|
||||
act_input.view(num_experts, max_num_tokens, -1),
|
||||
expert_tokens_meta.expert_num_tokens,
|
||||
beta,
|
||||
-1.0 if linear_beta is None else linear_beta,
|
||||
)
|
||||
|
||||
return batched_fused_marlin_moe(
|
||||
hidden_states=hidden_states,
|
||||
expert_num_tokens=expert_tokens_meta.expert_num_tokens,
|
||||
@@ -1051,4 +1111,7 @@ class BatchedMarlinExperts(MarlinExpertsBase):
|
||||
clamp_limit=self.gemm1_clamp_limit,
|
||||
gemm1_alpha=self.gemm1_alpha,
|
||||
gemm1_beta=self.gemm1_beta,
|
||||
activation_func=activation_func,
|
||||
activation_situ_beta=self.moe_config.activation_situ_beta,
|
||||
activation_situ_linear_beta=self.moe_config.activation_situ_linear_beta,
|
||||
)
|
||||
|
||||
@@ -79,6 +79,36 @@ class TrtLlmMxfp4ExpertsBase:
|
||||
else:
|
||||
self.gemm1_clamp_limit = None
|
||||
|
||||
# SITU (SituGLU) TRTLLM-Gen kernel computes
|
||||
# left = alpha * tanh(x0 / alpha) * sigmoid(x0) # gate (x0)
|
||||
# right = beta * tanh(x1 / beta) # up (x1)
|
||||
# which matches vLLM's situ_and_mul with (beta, linear_beta), so map
|
||||
# situ beta -> gatedActAlpha (gemm1_alpha) and situ linear_beta ->
|
||||
# gatedActBeta (gemm1_beta). Both must be > 0.
|
||||
if moe_config.activation == MoEActivation.SITU:
|
||||
situ_beta = moe_config.activation_situ_beta
|
||||
situ_linear_beta = moe_config.activation_situ_linear_beta
|
||||
assert situ_beta is not None and situ_beta > 0, (
|
||||
"SITU requires activation_situ_beta > 0"
|
||||
)
|
||||
assert situ_linear_beta is not None and situ_linear_beta > 0, (
|
||||
"TRTLLM SiTuGlu requires activation_situ_linear_beta > 0 "
|
||||
"(the private cubin has no up-passthrough path)"
|
||||
)
|
||||
self.gemm1_alpha = torch.full(
|
||||
(self.local_num_experts,),
|
||||
float(situ_beta),
|
||||
dtype=torch.float32,
|
||||
device=device,
|
||||
)
|
||||
self.gemm1_beta = torch.full(
|
||||
(self.local_num_experts,),
|
||||
float(situ_linear_beta),
|
||||
dtype=torch.float32,
|
||||
device=device,
|
||||
)
|
||||
self.gemm1_clamp_limit = None
|
||||
|
||||
self.max_capture_size = moe_config.max_capture_size
|
||||
|
||||
@staticmethod
|
||||
@@ -103,7 +133,19 @@ class TrtLlmMxfp4ExpertsBase:
|
||||
|
||||
@staticmethod
|
||||
def _supports_activation(activation: MoEActivation) -> bool:
|
||||
return activation in (MoEActivation.SWIGLUOAI, MoEActivation.SILU)
|
||||
return activation in (
|
||||
MoEActivation.SWIGLUOAI,
|
||||
MoEActivation.SILU,
|
||||
MoEActivation.SITU,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _flashinfer_activation_type(activation: MoEActivation) -> int:
|
||||
from flashinfer.fused_moe.core import ActivationType
|
||||
|
||||
if activation == MoEActivation.SITU:
|
||||
return ActivationType.Situ.value
|
||||
return ActivationType.Swiglu.value
|
||||
|
||||
@staticmethod
|
||||
def activation_format() -> mk.FusedMoEActivationFormat:
|
||||
@@ -142,6 +184,7 @@ class TrtLlmMxfp4ExpertsMonolithic(
|
||||
activation_key: QuantKey | None,
|
||||
) -> bool:
|
||||
return routing_method in [
|
||||
RoutingMethodType.DeepSeekV3,
|
||||
RoutingMethodType.Renormalize,
|
||||
RoutingMethodType.RenormalizeNaive,
|
||||
]
|
||||
@@ -192,8 +235,8 @@ class TrtLlmMxfp4ExpertsMonolithic(
|
||||
device=hidden_states.device,
|
||||
)
|
||||
trtllm_fp4_block_scale_moe(
|
||||
routing_logits=router_logits.to(torch.bfloat16),
|
||||
routing_bias=None,
|
||||
routing_logits=router_logits,
|
||||
routing_bias=e_score_correction_bias,
|
||||
hidden_states=x_quant,
|
||||
hidden_states_scale=x_scale,
|
||||
gemm1_weights=w1,
|
||||
@@ -210,14 +253,15 @@ class TrtLlmMxfp4ExpertsMonolithic(
|
||||
output2_scale_scalar=None,
|
||||
num_experts=global_num_experts,
|
||||
top_k=self.topk,
|
||||
n_group=None,
|
||||
topk_group=None,
|
||||
n_group=(num_expert_group or 0),
|
||||
topk_group=(topk_group or 0),
|
||||
intermediate_size=self.intermediate_size_per_partition,
|
||||
local_expert_offset=self.ep_rank * self.local_num_experts,
|
||||
local_num_experts=self.local_num_experts,
|
||||
routed_scaling_factor=None,
|
||||
routed_scaling_factor=routed_scaling_factor,
|
||||
routing_method_type=self.routing_method_type,
|
||||
do_finalize=True,
|
||||
activation_type=self._flashinfer_activation_type(activation),
|
||||
tune_max_num_tokens=max(self.max_capture_size, 1),
|
||||
output=output,
|
||||
routing_replay_out=routing_replay_out,
|
||||
@@ -338,6 +382,7 @@ class TrtLlmMxfp4ExpertsModular(TrtLlmMxfp4ExpertsBase, mk.FusedMoEExpertsModula
|
||||
"routing_method_type": RoutingMethodType.Renormalize,
|
||||
"do_finalize": True,
|
||||
"enable_pdl": True,
|
||||
"activation_type": self._flashinfer_activation_type(activation),
|
||||
"output": output,
|
||||
"tune_max_num_tokens": max(self.max_capture_size, 1),
|
||||
}
|
||||
|
||||
@@ -110,6 +110,27 @@ class TrtLlmNvFp4ExpertsBase:
|
||||
self.gemm1_alpha = None
|
||||
self.gemm1_beta = None
|
||||
|
||||
# SITU (Kimi SituGLU) TRTLLM-Gen kernel computes
|
||||
# left=alpha*tanh(x0/alpha)*sigmoid(x0), right=beta*tanh(x1/beta),
|
||||
# matching vLLM's situ_and_mul, so map situ beta -> gatedActAlpha
|
||||
# (gemm1_alpha) and situ linear_beta -> gatedActBeta (gemm1_beta).
|
||||
# These operate on the dequantized gate/up, so they are NOT folded by
|
||||
# g1_alphas in process_weights_after_loading.
|
||||
self.is_situ = moe_config.activation == MoEActivation.SITU
|
||||
if self.is_situ:
|
||||
situ_beta = moe_config.activation_situ_beta
|
||||
situ_linear_beta = moe_config.activation_situ_linear_beta
|
||||
assert situ_beta is not None and situ_beta > 0, (
|
||||
"SITU requires activation_situ_beta > 0"
|
||||
)
|
||||
assert situ_linear_beta is not None and situ_linear_beta > 0, (
|
||||
"TRTLLM SiTuGlu requires activation_situ_linear_beta > 0 "
|
||||
"(the private cubin has no up-passthrough path)"
|
||||
)
|
||||
self.gemm1_alpha = _per_expert(situ_beta)
|
||||
self.gemm1_beta = _per_expert(situ_linear_beta)
|
||||
self.gemm1_clamp_limit = None
|
||||
|
||||
logger.debug_once(
|
||||
"activation=%s, gemm1_alpha=%s, gemm1_beta=%s, gemm1_clamp_limit=%s",
|
||||
moe_config.activation,
|
||||
@@ -142,7 +163,10 @@ class TrtLlmNvFp4ExpertsBase:
|
||||
# (via the in-place mul above) and never changes again, so this is a
|
||||
# static, per-expert constant. Register on the layer so EPLB
|
||||
# rearranges it alongside the other expert tensors.
|
||||
if self.gemm1_clamp_limit is not None:
|
||||
# SITU alpha/beta act on the dequantized gate/up (tanh clamps), not the
|
||||
# raw GEMM1 accumulator, so they are registered as-is without the
|
||||
# g1_alphas fold used by the SwiGLU-OAI clamp/beta below.
|
||||
if self.gemm1_clamp_limit is not None and not self.is_situ:
|
||||
gemm1_clamp_limit = self.gemm1_clamp_limit / self.quant_config.g1_alphas
|
||||
layer.register_parameter(
|
||||
"gemm1_clamp_limit",
|
||||
@@ -155,7 +179,11 @@ class TrtLlmNvFp4ExpertsBase:
|
||||
# raw. Register both on the layer so EPLB rearranges them with the
|
||||
# other per-expert tensors.
|
||||
if self.gemm1_beta is not None:
|
||||
gemm1_beta = self.gemm1_beta / self.quant_config.g1_alphas
|
||||
gemm1_beta = (
|
||||
self.gemm1_beta
|
||||
if self.is_situ
|
||||
else self.gemm1_beta / self.quant_config.g1_alphas
|
||||
)
|
||||
layer.register_parameter(
|
||||
"gemm1_beta",
|
||||
torch.nn.Parameter(gemm1_beta, requires_grad=False),
|
||||
@@ -197,13 +225,15 @@ class TrtLlmNvFp4ExpertsBase:
|
||||
|
||||
@staticmethod
|
||||
def _supports_activation(activation: MoEActivation) -> bool:
|
||||
"""Supports SiLU, RELU^2 non-gated, GELU, and clamped SwiGLU-OAI."""
|
||||
"""Supports SiLU, RELU^2 non-gated, GELU, clamped SwiGLU-OAI, and the
|
||||
private SituGLU (SITU) cubin."""
|
||||
return activation in [
|
||||
MoEActivation.SILU,
|
||||
MoEActivation.RELU2_NO_MUL,
|
||||
MoEActivation.GELU,
|
||||
MoEActivation.GELU_TANH,
|
||||
MoEActivation.SWIGLUOAI_UNINTERLEAVE,
|
||||
MoEActivation.SITU,
|
||||
]
|
||||
|
||||
@staticmethod
|
||||
|
||||
@@ -120,6 +120,8 @@ def FusedMoE(
|
||||
swiglu_limit: float | None = None,
|
||||
swiglu_alpha: float | None = None,
|
||||
swiglu_beta: float | None = None,
|
||||
activation_situ_beta: float | None = None,
|
||||
activation_situ_linear_beta: float | None = None,
|
||||
e_score_correction_bias: torch.Tensor | None = None,
|
||||
apply_router_weight_on_input: bool = False,
|
||||
activation: str = "silu",
|
||||
@@ -178,6 +180,8 @@ def FusedMoE(
|
||||
scoring_func: Scoring function for routing ("softmax" or others)
|
||||
routed_scaling_factor: Scaling factor applied to topk_weights or output
|
||||
swiglu_limit: SwiGLU activation limit
|
||||
activation_situ_beta: SituGLU activation beta
|
||||
activation_situ_linear_beta: SituGLU linear beta
|
||||
e_score_correction_bias: Expert score correction bias tensor
|
||||
apply_router_weight_on_input: Whether to apply router weights on input
|
||||
activation: Activation function name ("silu", "gelu", etc.)
|
||||
@@ -352,6 +356,8 @@ def FusedMoE(
|
||||
swiglu_limit=swiglu_limit,
|
||||
swiglu_alpha=swiglu_alpha,
|
||||
swiglu_beta=swiglu_beta,
|
||||
activation_situ_beta=activation_situ_beta,
|
||||
activation_situ_linear_beta=activation_situ_linear_beta,
|
||||
max_capture_size=vllm_config.compilation_config.max_cudagraph_capture_size,
|
||||
skip_final_all_reduce=skip_final_all_reduce,
|
||||
)
|
||||
|
||||
@@ -890,6 +890,8 @@ class FusedMoEExpertsModular(FusedMoEExperts):
|
||||
beta: float = 0.0,
|
||||
topk_ids: torch.Tensor | None = None,
|
||||
expert_map: torch.Tensor | None = None,
|
||||
activation_situ_beta: float | None = None,
|
||||
activation_situ_linear_beta: float | None = None,
|
||||
) -> None:
|
||||
apply_moe_activation(
|
||||
activation,
|
||||
@@ -900,6 +902,16 @@ class FusedMoEExpertsModular(FusedMoEExperts):
|
||||
beta=beta,
|
||||
topk_ids=topk_ids,
|
||||
expert_map=expert_map,
|
||||
activation_situ_beta=(
|
||||
self.moe_config.activation_situ_beta
|
||||
if activation_situ_beta is None
|
||||
else activation_situ_beta
|
||||
),
|
||||
activation_situ_linear_beta=(
|
||||
self.moe_config.activation_situ_linear_beta
|
||||
if activation_situ_linear_beta is None
|
||||
else activation_situ_linear_beta
|
||||
),
|
||||
)
|
||||
|
||||
@abstractmethod
|
||||
@@ -1307,6 +1319,14 @@ class FusedMoEKernelModularImpl:
|
||||
activation,
|
||||
)
|
||||
|
||||
use_output_alias = (
|
||||
output_alias is not None
|
||||
and output_alias.shape == fused_out.shape
|
||||
and output_alias.dtype == fused_out.dtype
|
||||
and output_alias.device == fused_out.device
|
||||
and output_alias.is_contiguous()
|
||||
)
|
||||
|
||||
# If caller's output buffer already matches fused_out shape/dtype, alias
|
||||
# to skip the redundant copy in TopKWeightAndReduceNoOP.apply downstream.
|
||||
# This eliminates ~94% of __amd_rocclr_copyBuffer events (Copy 2 of the
|
||||
@@ -1314,15 +1334,10 @@ class FusedMoEKernelModularImpl:
|
||||
if current_platform.is_rocm():
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
|
||||
if (
|
||||
rocm_aiter_ops.is_fused_moe_enabled()
|
||||
and output_alias is not None
|
||||
and output_alias.shape == fused_out.shape
|
||||
and output_alias.dtype == fused_out.dtype
|
||||
and output_alias.device == fused_out.device
|
||||
and output_alias.is_contiguous()
|
||||
):
|
||||
if use_output_alias and rocm_aiter_ops.is_fused_moe_enabled():
|
||||
fused_out = output_alias
|
||||
elif use_output_alias:
|
||||
fused_out = output_alias
|
||||
|
||||
self.fused_experts.apply(
|
||||
output=fused_out,
|
||||
|
||||
@@ -0,0 +1,255 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import torch
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.config import get_current_vllm_config
|
||||
from vllm.distributed import tensor_model_parallel_all_reduce
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.fused_allreduce_gemma_rms_norm import (
|
||||
_AR_RESIDUAL_RMS_NORM,
|
||||
_can_use_flashinfer,
|
||||
flashinfer_trtllm_fused_allreduce_norm,
|
||||
)
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.utils.torch_utils import aux_stream, current_stream
|
||||
|
||||
from .moe_runner import MoERunner, _unpack
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class LatentMoERunner(MoERunner):
|
||||
"""MoE runner for latent MoE with a replicated routed up-projection.
|
||||
|
||||
Fused path (tp>1, un-reduced combine output, shared expert, no SP):
|
||||
concatenates the un-reduced latent partial (dim d) and the un-reduced
|
||||
shared partial (dim D) into one contiguous buffer, all-reduces once, then
|
||||
splits. The latent half is normed and up-projected locally (replicated
|
||||
up-proj -> full hidden), and the shared add folds into the GEMM epilogue
|
||||
(``torch.addmm``). One collective total, no post-reduction communication.
|
||||
|
||||
Native path: the replicated up-proj produces the full hidden dim on every
|
||||
rank, so the base runner combines routed + shared correctly at any TP size
|
||||
(using two collectives instead of the fused path's one).
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*args,
|
||||
enable_k3_latent_moe_tail_fusion: bool = False,
|
||||
**kwargs,
|
||||
) -> None:
|
||||
super().__init__(*args, **kwargs)
|
||||
self.enable_k3_latent_moe_tail_fusion = enable_k3_latent_moe_tail_fusion
|
||||
use_fused_path = self._use_fused_path()
|
||||
if (
|
||||
self.enable_k3_latent_moe_tail_fusion
|
||||
and use_fused_path
|
||||
and self.moe_config.tp_size not in (8, 16)
|
||||
):
|
||||
logger.warning_once(
|
||||
"K3 latent-MoE tail fusion currently supports TP=8 and TP=16, "
|
||||
"but TP=%d is configured. Falling back to the default path.",
|
||||
self.moe_config.tp_size,
|
||||
)
|
||||
self.enable_k3_latent_moe_tail_fusion = False
|
||||
|
||||
if self.enable_k3_latent_moe_tail_fusion and use_fused_path:
|
||||
vllm_config = get_current_vllm_config()
|
||||
if vllm_config.parallel_config.use_ubatching:
|
||||
raise ValueError(
|
||||
"K3 latent-MoE tail fusion does not support DBO or ubatching."
|
||||
)
|
||||
if vllm_config.model_config.enable_sleep_mode:
|
||||
raise ValueError(
|
||||
"K3 latent-MoE tail fusion does not support sleep mode."
|
||||
)
|
||||
transform = self.routed_output_transform
|
||||
assert transform is not None
|
||||
norm = transform.norm
|
||||
assert norm is not None
|
||||
from vllm.models.kimi_k3.nvidia.ops.latent_moe_tail import (
|
||||
KimiK3LatentMoETailOp,
|
||||
)
|
||||
|
||||
op = KimiK3LatentMoETailOp.initialize(
|
||||
hidden_size=transform.up_proj.weight.shape[0],
|
||||
latent_size=norm.weight.shape[0],
|
||||
dtype=norm.weight.dtype,
|
||||
device=norm.weight.device,
|
||||
rms_eps=norm.variance_epsilon,
|
||||
)
|
||||
self._k3_latent_moe_tail_op = op
|
||||
|
||||
def _get_zero_residual(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
||||
"""Read-only zero ``residual_in`` for the fused AR+RMSNorm kernel.
|
||||
|
||||
flashinfer requires a residual buffer even when there is no residual to
|
||||
add.
|
||||
"""
|
||||
numel = hidden_states.numel()
|
||||
buf = getattr(self, "_zero_residual", None)
|
||||
if (
|
||||
buf is None
|
||||
or buf.dtype != hidden_states.dtype
|
||||
or buf.device != hidden_states.device
|
||||
or buf.numel() < numel
|
||||
):
|
||||
buf = torch.zeros(
|
||||
numel, dtype=hidden_states.dtype, device=hidden_states.device
|
||||
)
|
||||
self._zero_residual = buf
|
||||
return buf[:numel].view_as(hidden_states)
|
||||
|
||||
def _use_fused_path(self) -> bool:
|
||||
# The fused path merges the latent and shared reductions into one
|
||||
# all-reduce, so it needs actual TP parallelism, a shared expert (to
|
||||
# concat), an un-reduced combine output, and no sequence parallelism.
|
||||
return (
|
||||
self.moe_config.tp_size > 1
|
||||
and self._shared_experts is not None
|
||||
and not self._fused_output_is_reduced
|
||||
and not self.moe_config.is_sequence_parallel
|
||||
)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
router_logits: torch.Tensor,
|
||||
input_ids: torch.Tensor | None = None,
|
||||
shared_experts_input: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
if self._use_fused_path():
|
||||
return self._fused_forward(
|
||||
hidden_states, router_logits, input_ids, shared_experts_input
|
||||
)
|
||||
return super().forward(
|
||||
hidden_states, router_logits, input_ids, shared_experts_input
|
||||
)
|
||||
|
||||
def _fused_forward(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
router_logits: torch.Tensor,
|
||||
input_ids: torch.Tensor | None = None,
|
||||
shared_experts_input: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
# When the caller pre-applies the routed input transform outside the
|
||||
# runner (e.g. to overlap it on a separate stream), it passes the
|
||||
# already-transformed routed input as ``hidden_states`` and the original
|
||||
# hidden states as ``shared_experts_input``; skip the transform then.
|
||||
if shared_experts_input is None:
|
||||
hidden_states, shared_experts_input = self.apply_routed_input_transform(
|
||||
hidden_states
|
||||
)
|
||||
hidden_states, og_hidden_dim_pre_xform, og_hidden_dim_post_xform = (
|
||||
self._maybe_pad_hidden_states(
|
||||
shared_experts_input,
|
||||
hidden_states,
|
||||
)
|
||||
)
|
||||
|
||||
result = self._forward_entry(
|
||||
hidden_states,
|
||||
router_logits,
|
||||
shared_experts_input,
|
||||
input_ids,
|
||||
self._encode_layer_name(),
|
||||
self.moe_config.hidden_dim_unpadded
|
||||
if self._quant_method.has_unpadded_output
|
||||
else 0,
|
||||
)
|
||||
|
||||
shared_output, fused_output = _unpack(result)
|
||||
assert shared_output is not None
|
||||
|
||||
if og_hidden_dim_pre_xform is not None:
|
||||
fused_output = fused_output[..., :og_hidden_dim_pre_xform]
|
||||
|
||||
transform = self.routed_output_transform
|
||||
assert transform is not None
|
||||
|
||||
if self.enable_k3_latent_moe_tail_fusion:
|
||||
op = self._k3_latent_moe_tail_op
|
||||
if 0 < fused_output.shape[0] <= op.contract.max_num_tokens:
|
||||
norm = transform.norm
|
||||
assert norm is not None
|
||||
result = op(
|
||||
fused_output,
|
||||
shared_output,
|
||||
norm.weight,
|
||||
transform.up_proj.weight,
|
||||
)
|
||||
result = self._maybe_reduce_final_output(
|
||||
result, og_hidden_dim_post_xform, output_is_reduced=True
|
||||
)
|
||||
return self._maybe_add_zero_expert_output(result)
|
||||
|
||||
fused_latent = None
|
||||
if transform.norm is not None:
|
||||
fused_latent = self.allreduce_norm_latent_out(fused_output, transform.norm)
|
||||
else:
|
||||
fused_latent = tensor_model_parallel_all_reduce(fused_output)
|
||||
|
||||
shared_expert_stream = (
|
||||
aux_stream()
|
||||
if shared_output.size(0) <= envs.VLLM_SHARED_EXPERTS_STREAM_TOKEN_THRESHOLD
|
||||
else None
|
||||
)
|
||||
if shared_expert_stream is not None:
|
||||
# overlap shared expert allreduce with latent up_proj
|
||||
main = current_stream()
|
||||
shared_output.record_stream(shared_expert_stream)
|
||||
shared_expert_stream.wait_stream(main)
|
||||
with torch.cuda.stream(shared_expert_stream):
|
||||
shared_output = tensor_model_parallel_all_reduce(shared_output)
|
||||
result = torch.mm(fused_latent, transform.up_proj.weight.t())
|
||||
main.wait_stream(shared_expert_stream)
|
||||
else:
|
||||
shared_output = tensor_model_parallel_all_reduce(shared_output)
|
||||
result = torch.mm(fused_latent, transform.up_proj.weight.t())
|
||||
result.add_(shared_output)
|
||||
|
||||
# Output is already fully reduced; this only strips padding.
|
||||
result = self._maybe_reduce_final_output(
|
||||
result, og_hidden_dim_post_xform, output_is_reduced=True
|
||||
)
|
||||
|
||||
return self._maybe_add_zero_expert_output(result)
|
||||
|
||||
def allreduce_norm_latent_out(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
norm: RMSNorm,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""All-reduce + add residual + (standard) RMSNorm, fused via flashinfer."""
|
||||
if self.moe_config.tp_size == 1:
|
||||
return norm(hidden_states)
|
||||
|
||||
if flashinfer_trtllm_fused_allreduce_norm is not None:
|
||||
ok, max_token_num = _can_use_flashinfer(
|
||||
hidden_states, self.moe_config.tp_size
|
||||
)
|
||||
if ok:
|
||||
norm_out = torch.empty_like(hidden_states)
|
||||
# With norm_out provided, the kernel writes the new residual
|
||||
# (all_reduce(hidden_states) + residual) into the hidden_states
|
||||
# buffer and the normalized result into norm_out.
|
||||
flashinfer_trtllm_fused_allreduce_norm(
|
||||
allreduce_in=hidden_states,
|
||||
residual=self._get_zero_residual(hidden_states),
|
||||
rms_gamma=norm.weight,
|
||||
rms_eps=norm.variance_epsilon,
|
||||
world_size=self.moe_config.tp_size,
|
||||
weight_bias=0.0,
|
||||
launch_with_pdl=True,
|
||||
fp32_acc=True,
|
||||
max_token_num=max_token_num,
|
||||
pattern_code=_AR_RESIDUAL_RMS_NORM,
|
||||
norm_out=norm_out,
|
||||
)
|
||||
return norm_out
|
||||
|
||||
reduced = tensor_model_parallel_all_reduce(hidden_states)
|
||||
return norm(reduced)
|
||||
@@ -348,7 +348,8 @@ class MoERunner(MoERunnerInterface):
|
||||
return self.routed_experts.quant_method
|
||||
|
||||
def apply_routed_input_transform(
|
||||
self, hidden_states: torch.Tensor
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor | None]:
|
||||
"""Apply transform for routed experts (e.g., latent projection).
|
||||
|
||||
@@ -416,6 +417,7 @@ class MoERunner(MoERunnerInterface):
|
||||
def _maybe_reduce_shared_expert_output(
|
||||
self,
|
||||
shared_output: torch.Tensor | None,
|
||||
fused_output_is_reduced: bool | None = None,
|
||||
) -> torch.Tensor | None:
|
||||
"""All-reduce shared expert output when the combine kernel already
|
||||
reduced fused output.
|
||||
@@ -425,18 +427,43 @@ class MoERunner(MoERunnerInterface):
|
||||
* If we have SP (TP=N, DP=M, EP), there is a separate AG step handled
|
||||
in the model.
|
||||
"""
|
||||
if fused_output_is_reduced is None:
|
||||
fused_output_is_reduced = self._fused_output_is_reduced
|
||||
|
||||
if (
|
||||
shared_output is not None
|
||||
and not self.moe_config.is_sequence_parallel
|
||||
and self._fused_output_is_reduced
|
||||
and fused_output_is_reduced
|
||||
):
|
||||
shared_output = tensor_model_parallel_all_reduce(shared_output)
|
||||
return shared_output
|
||||
|
||||
def _maybe_reduce_routed_output_before_transform(
|
||||
self,
|
||||
fused_output: torch.Tensor,
|
||||
fused_output_is_reduced: bool,
|
||||
) -> tuple[torch.Tensor, bool]:
|
||||
"""All-reduce latent routed output before its output transform.
|
||||
|
||||
Latent MoE output transforms may contain non-linear ops, e.g. RMSNorm.
|
||||
TP partial routed outputs must be summed in latent space before such
|
||||
transforms are applied.
|
||||
"""
|
||||
if (
|
||||
self.routed_output_transform is not None
|
||||
and not self.moe_config.is_sequence_parallel
|
||||
and (self.moe_config.tp_size > 1 or self.moe_config.ep_size > 1)
|
||||
and not fused_output_is_reduced
|
||||
):
|
||||
fused_output = tensor_model_parallel_all_reduce(fused_output)
|
||||
fused_output_is_reduced = True
|
||||
return fused_output, fused_output_is_reduced
|
||||
|
||||
def _maybe_reduce_final_output(
|
||||
self,
|
||||
states: torch.Tensor,
|
||||
trunc_size: int | None,
|
||||
output_is_reduced: bool | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""All-reduce the combined output if needed.
|
||||
|
||||
@@ -455,11 +482,14 @@ class MoERunner(MoERunnerInterface):
|
||||
# We don't need to reduce the final output if:
|
||||
# - We are not running with TP or DP
|
||||
# - The MK already reduced the fused output itself.
|
||||
if output_is_reduced is None:
|
||||
output_is_reduced = self._fused_output_is_reduced
|
||||
|
||||
if (
|
||||
not self.moe_config.is_sequence_parallel
|
||||
and not self.moe_config.skip_final_all_reduce
|
||||
and (self.moe_config.tp_size > 1 or self.moe_config.ep_size > 1)
|
||||
and not self._fused_output_is_reduced
|
||||
and not output_is_reduced
|
||||
):
|
||||
states = tensor_model_parallel_all_reduce(states)
|
||||
|
||||
@@ -643,6 +673,7 @@ class MoERunner(MoERunnerInterface):
|
||||
hidden_states: torch.Tensor,
|
||||
router_logits: torch.Tensor,
|
||||
input_ids: torch.Tensor | None = None,
|
||||
shared_experts_input: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""Invoke the fused moe layer.
|
||||
|
||||
@@ -664,11 +695,16 @@ class MoERunner(MoERunnerInterface):
|
||||
_moe_forward and _moe_forward_shared must be split.
|
||||
"""
|
||||
|
||||
# Apply transform for routed experts (e.g., latent projection
|
||||
# for latent MoE)
|
||||
hidden_states, shared_experts_input = self.apply_routed_input_transform(
|
||||
hidden_states
|
||||
)
|
||||
# Apply transform for routed experts (e.g., latent projection for
|
||||
# latent MoE). When the caller pre-applies the routed input transform
|
||||
# outside the runner (e.g. to overlap it on a separate stream), it
|
||||
# passes the already-transformed routed input as ``hidden_states`` and
|
||||
# the original hidden states as ``shared_experts_input``; skip the
|
||||
# transform in that case so shared experts still see the original input.
|
||||
if shared_experts_input is None:
|
||||
hidden_states, shared_experts_input = self.apply_routed_input_transform(
|
||||
hidden_states
|
||||
)
|
||||
|
||||
# Record before `_maybe_pad_hidden_states` pads activations to match
|
||||
# `moe_config.hidden_dim`, e.g. after `align_trtllm_fp4_moe_hidden_dim_for_fi`
|
||||
@@ -708,9 +744,22 @@ class MoERunner(MoERunnerInterface):
|
||||
if og_hidden_dim_pre_xform is not None:
|
||||
fused_output = fused_output[..., :og_hidden_dim_pre_xform]
|
||||
|
||||
# If combine kernel already reduced fused, reduce shared to match.
|
||||
fused_output_is_reduced = self._fused_output_is_reduced
|
||||
|
||||
# Latent routed output has to be reduced before output transform,
|
||||
# because the transform may include non-linear normalization.
|
||||
fused_output, fused_output_is_reduced = (
|
||||
self._maybe_reduce_routed_output_before_transform(
|
||||
fused_output,
|
||||
fused_output_is_reduced,
|
||||
)
|
||||
)
|
||||
|
||||
# If routed output is already reduced, reduce shared to match.
|
||||
# See note above re: the two all-reduce points.
|
||||
shared_output = self._maybe_reduce_shared_expert_output(shared_output)
|
||||
shared_output = self._maybe_reduce_shared_expert_output(
|
||||
shared_output, fused_output_is_reduced
|
||||
)
|
||||
|
||||
shared_output, fused_output = self._maybe_apply_routed_scale_to_output(
|
||||
shared_output, fused_output
|
||||
@@ -724,7 +773,9 @@ class MoERunner(MoERunnerInterface):
|
||||
else:
|
||||
result = fused_output
|
||||
|
||||
result = self._maybe_reduce_final_output(result, og_hidden_dim_post_xform)
|
||||
result = self._maybe_reduce_final_output(
|
||||
result, og_hidden_dim_post_xform, fused_output_is_reduced
|
||||
)
|
||||
|
||||
return self._maybe_add_zero_expert_output(result)
|
||||
|
||||
|
||||
@@ -36,6 +36,7 @@ class MoERunnerInterface(PluggableLayer, ABC):
|
||||
hidden_states: torch.Tensor,
|
||||
router_logits: torch.Tensor,
|
||||
input_ids: torch.Tensor | None = None,
|
||||
shared_experts_input: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
@@ -1080,6 +1080,7 @@ def causal_conv1d_update(
|
||||
block_idx_last_scheduled_token: torch.Tensor | None = None,
|
||||
initial_state_idx: torch.Tensor | None = None,
|
||||
validate_data=False,
|
||||
out: torch.Tensor | None = None,
|
||||
):
|
||||
"""
|
||||
x: Input tensor which can take the following shapes:
|
||||
@@ -1117,7 +1118,8 @@ def causal_conv1d_update(
|
||||
for example: conv_state_indices = [null_block_id, 1, 20, null_block_id]
|
||||
in this case, the kernel will not process entries at
|
||||
indices 0 and 3
|
||||
out: (batch, dim) or (batch, dim, seqlen) or (num_tokens, dim), same shape as `x`
|
||||
out: optional output tensor with the same shape as `x`. When omitted,
|
||||
the input is overwritten.
|
||||
"""
|
||||
if validate_data:
|
||||
assert null_block_id is not None
|
||||
@@ -1129,10 +1131,22 @@ def causal_conv1d_update(
|
||||
|
||||
original_x_dtype = x.dtype
|
||||
x = x.to(conv_state.dtype)
|
||||
if out is None:
|
||||
out = x
|
||||
else:
|
||||
if out.shape != x.shape:
|
||||
raise ValueError(
|
||||
f"`out` shape {tuple(out.shape)} must match `x` shape {tuple(x.shape)}."
|
||||
)
|
||||
if out.dtype != original_x_dtype or out.device != x.device:
|
||||
raise ValueError(
|
||||
"`out` must have the same dtype and device as the input `x`."
|
||||
)
|
||||
unsqueeze = query_start_loc is None and x.dim() == 2
|
||||
if unsqueeze:
|
||||
# make it (batch, dim, seqlen) with seqlen == 1
|
||||
x = x.unsqueeze(-1)
|
||||
out = out.unsqueeze(-1)
|
||||
if query_start_loc is None:
|
||||
batch, dim, seqlen = x.shape
|
||||
else:
|
||||
@@ -1159,8 +1173,6 @@ def causal_conv1d_update(
|
||||
assert num_cache_lines >= batch
|
||||
assert weight.stride(1) == 1 # Need this
|
||||
|
||||
# adopt the strategy in vLLM that overwrite on 'x' directly, rather than creating a new tensor 'o'
|
||||
out = x
|
||||
stride_w_dim, stride_w_width = weight.stride()
|
||||
|
||||
if query_start_loc is None:
|
||||
|
||||
@@ -0,0 +1,83 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import tl, triton
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _gather_initial_states_kernel(
|
||||
state_ptr,
|
||||
indices_ptr,
|
||||
has_initial_state_ptr,
|
||||
output_ptr,
|
||||
stride_state_batch,
|
||||
stride_indices,
|
||||
stride_has_initial_state,
|
||||
row_size: tl.constexpr,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
launch_pdl: tl.constexpr,
|
||||
):
|
||||
block_idx = tl.program_id(0)
|
||||
batch_idx = tl.program_id(1)
|
||||
offsets = block_idx * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)
|
||||
mask = offsets < row_size
|
||||
|
||||
if launch_pdl:
|
||||
tl.extra.cuda.gdc_wait()
|
||||
tl.extra.cuda.gdc_launch_dependents()
|
||||
|
||||
has_initial_state = tl.load(
|
||||
has_initial_state_ptr + batch_idx * stride_has_initial_state
|
||||
).to(tl.int1)
|
||||
state_idx = tl.load(indices_ptr + batch_idx * stride_indices).to(tl.int64)
|
||||
state_idx = tl.where(has_initial_state, state_idx, 0)
|
||||
values = tl.load(
|
||||
state_ptr + state_idx * stride_state_batch + offsets,
|
||||
mask=mask & has_initial_state,
|
||||
other=0.0,
|
||||
)
|
||||
tl.store(output_ptr + batch_idx * row_size + offsets, values, mask=mask)
|
||||
|
||||
|
||||
def gather_initial_states(
|
||||
state: torch.Tensor,
|
||||
indices: torch.Tensor,
|
||||
has_initial_state: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
"""Gather dense state rows, replacing uninitialized rows with zeros."""
|
||||
assert state.ndim >= 2
|
||||
assert state.is_cuda
|
||||
assert indices.ndim == 1 and has_initial_state.ndim == 1
|
||||
assert indices.shape == has_initial_state.shape
|
||||
assert indices.device == state.device
|
||||
assert has_initial_state.device == state.device
|
||||
assert indices.dtype in (torch.int32, torch.int64)
|
||||
assert has_initial_state.dtype == torch.bool
|
||||
|
||||
row_size = state[0].numel()
|
||||
# Mamba pages may pad stride(0), but each state row remains dense.
|
||||
assert state[0].is_contiguous()
|
||||
output = torch.empty(
|
||||
(indices.numel(), *state.shape[1:]),
|
||||
dtype=state.dtype,
|
||||
device=state.device,
|
||||
)
|
||||
block_size = min(triton.next_power_of_2(row_size), 1024)
|
||||
grid = (triton.cdiv(row_size, block_size), indices.numel())
|
||||
_gather_initial_states_kernel[grid](
|
||||
state,
|
||||
indices,
|
||||
has_initial_state,
|
||||
output,
|
||||
state.stride(0),
|
||||
indices.stride(0),
|
||||
has_initial_state.stride(0),
|
||||
row_size=row_size,
|
||||
BLOCK_SIZE=block_size,
|
||||
num_warps=8,
|
||||
launch_pdl=current_platform.is_arch_support_pdl(),
|
||||
)
|
||||
return output
|
||||
@@ -0,0 +1,2 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
@@ -0,0 +1,9 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Ops shared across model implementations."""
|
||||
|
||||
from .fused_qk_rmsnorm import fused_q_kv_rmsnorm
|
||||
|
||||
__all__ = [
|
||||
"fused_q_kv_rmsnorm",
|
||||
]
|
||||
@@ -0,0 +1,103 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import torch
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import tl, triton
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _fused_q_kv_rmsnorm_kernel(
|
||||
q_ptr,
|
||||
q_out_ptr,
|
||||
q_weight_ptr,
|
||||
q_in_stride,
|
||||
q_out_stride,
|
||||
kv_ptr,
|
||||
kv_out_ptr,
|
||||
kv_weight_ptr,
|
||||
kv_in_stride,
|
||||
kv_out_stride,
|
||||
eps,
|
||||
Q_SIZE: tl.constexpr,
|
||||
KV_SIZE: tl.constexpr,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
launch_pdl: tl.constexpr,
|
||||
):
|
||||
# num_tokens goes on grid-x (max 2**31 - 1); task goes on grid-y.
|
||||
# CUDA's grid-y/z are capped at 65535, so putting num_tokens there crashes
|
||||
# the launch at max-num-batched-tokens >= 65536 with "invalid argument".
|
||||
# int64: q_in_stride can be ~24K (128 heads × 192) and overflows int32
|
||||
# past num_tokens ~87K under large chunked prefill.
|
||||
token_idx = tl.program_id(0).to(tl.int64)
|
||||
pid_task = tl.program_id(1)
|
||||
|
||||
if pid_task == 0:
|
||||
SIZE = Q_SIZE
|
||||
row_in = q_ptr + token_idx * q_in_stride
|
||||
weight_ptr = q_weight_ptr
|
||||
row_out = q_out_ptr + token_idx * q_out_stride
|
||||
else:
|
||||
SIZE = KV_SIZE
|
||||
row_in = kv_ptr + token_idx * kv_in_stride
|
||||
weight_ptr = kv_weight_ptr
|
||||
row_out = kv_out_ptr + token_idx * kv_out_stride
|
||||
|
||||
if launch_pdl:
|
||||
tl.extra.cuda.gdc_wait()
|
||||
tl.extra.cuda.gdc_launch_dependents()
|
||||
|
||||
# RMSNorm in fp32 throughout — matches csrc/layernorm_kernels.cu's
|
||||
# `(scalar_t)(x * s_variance * w)` and DeepseekV4's compressor kernel, which
|
||||
# keep x, rrms, and w all in fp32 and perform a single cast at store.
|
||||
block = tl.arange(0, BLOCK_SIZE)
|
||||
mask = block < SIZE
|
||||
x = tl.load(row_in + block, mask=mask, other=0.0).to(tl.float32)
|
||||
variance = tl.sum(x * x, axis=0) / SIZE
|
||||
rrms = tl.rsqrt(variance + eps)
|
||||
w = tl.load(weight_ptr + block, mask=mask, other=0.0).to(tl.float32)
|
||||
y = x * rrms * w
|
||||
tl.store(row_out + block, y.to(row_out.dtype.element_ty), mask=mask)
|
||||
|
||||
|
||||
def fused_q_kv_rmsnorm(
|
||||
qr: torch.Tensor,
|
||||
kv: torch.Tensor,
|
||||
q_weight: torch.Tensor,
|
||||
kv_weight: torch.Tensor,
|
||||
eps: float,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
assert qr.ndim == 2 and kv.ndim == 2
|
||||
assert qr.shape[0] == kv.shape[0], (
|
||||
f"token dim mismatch: qr={qr.shape}, kv={kv.shape}"
|
||||
)
|
||||
assert qr.stride(-1) == 1 and kv.stride(-1) == 1
|
||||
assert q_weight.is_contiguous() and kv_weight.is_contiguous()
|
||||
|
||||
q_size = qr.shape[1]
|
||||
kv_size = kv.shape[1]
|
||||
num_tokens = qr.shape[0]
|
||||
qr_out = torch.empty_like(qr)
|
||||
kv_out = torch.empty_like(kv)
|
||||
if num_tokens == 0:
|
||||
return qr_out, kv_out
|
||||
|
||||
block_size = triton.next_power_of_2(max(q_size, kv_size))
|
||||
_fused_q_kv_rmsnorm_kernel[(num_tokens, 2)](
|
||||
qr,
|
||||
qr_out,
|
||||
q_weight,
|
||||
qr.stride(0),
|
||||
qr_out.stride(0),
|
||||
kv,
|
||||
kv_out,
|
||||
kv_weight,
|
||||
kv.stride(0),
|
||||
kv_out.stride(0),
|
||||
eps,
|
||||
Q_SIZE=q_size,
|
||||
KV_SIZE=kv_size,
|
||||
BLOCK_SIZE=block_size,
|
||||
launch_pdl=current_platform.is_arch_support_pdl(),
|
||||
)
|
||||
return qr_out, kv_out
|
||||
@@ -130,13 +130,20 @@ def inkling_fa4_rel_attention(
|
||||
cute_window = (None, None) if window_size == (-1, -1) else window_size
|
||||
|
||||
rel_logits = rel_logits.contiguous()
|
||||
flash_attn_varlen_func: Callable[..., Any]
|
||||
if _use_sheared_bias():
|
||||
from vllm.third_party.tml_fa4 import flash_attn_varlen_func
|
||||
from vllm.third_party.tml_fa4 import (
|
||||
flash_attn_varlen_func as tml_flash_attn_varlen_func,
|
||||
)
|
||||
|
||||
flash_attn_varlen_func = tml_flash_attn_varlen_func
|
||||
bias_kwargs: dict[str, Any] = {"rel_bias": rel_logits}
|
||||
else:
|
||||
from vllm.vllm_flash_attn.cute import flash_attn_varlen_func
|
||||
from vllm.vllm_flash_attn.cute import (
|
||||
flash_attn_varlen_func as cute_flash_attn_varlen_func,
|
||||
)
|
||||
|
||||
flash_attn_varlen_func = cute_flash_attn_varlen_func
|
||||
bias_kwargs = {
|
||||
"score_mod": _get_score_mod(rel_extent),
|
||||
"aux_tensors": [rel_logits],
|
||||
|
||||
@@ -1,2 +1,28 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Kimi K3 model — hardware-isolated entry point.
|
||||
|
||||
The implementation lives under ``nvidia/`` and ``amd/``; this module picks the
|
||||
right one for the current platform and re-exports the public classes used by
|
||||
the model registry. (Mirrors ``vllm.models.minimax_m3``.)
|
||||
"""
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
# The NVIDIA branch is the static default that type-checkers see; the ROCm
|
||||
# branch overrides it at runtime (kept type-compatible via type: ignore).
|
||||
if TYPE_CHECKING or not current_platform.is_rocm():
|
||||
from .nvidia.model import KimiK3ForConditionalGeneration, KimiLinearForCausalLM
|
||||
from .nvidia.mtp import KimiK3MTP
|
||||
else:
|
||||
from .amd.linear import KimiLinearForCausalLM # type: ignore[assignment]
|
||||
from .amd.model import KimiK3ForConditionalGeneration # type: ignore[assignment]
|
||||
from .amd.mtp import KimiK3MTP # type: ignore[assignment]
|
||||
|
||||
__all__ = [
|
||||
"KimiK3ForConditionalGeneration",
|
||||
"KimiK3MTP",
|
||||
"KimiLinearForCausalLM",
|
||||
]
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,249 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Kimi-K3 multimodal model implementation for vLLM."""
|
||||
|
||||
from collections.abc import Iterable
|
||||
from typing import cast
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
from vllm.model_executor.layers.quantization.compressed_tensors import (
|
||||
compressed_tensors,
|
||||
)
|
||||
from vllm.model_executor.models.interfaces import (
|
||||
HasInnerState,
|
||||
IsHybrid,
|
||||
SupportsEagle3,
|
||||
SupportsMultiModal,
|
||||
SupportsPP,
|
||||
SupportsQuant,
|
||||
)
|
||||
from vllm.model_executor.models.kimi_k25 import KimiK25MediaPixelInputs
|
||||
from vllm.model_executor.models.kimi_k25_vit import (
|
||||
KimiK25MultiModalProjector,
|
||||
MoonViT3dPretrainedModel,
|
||||
vision_tower_forward,
|
||||
)
|
||||
from vllm.model_executor.models.utils import (
|
||||
AutoWeightsLoader,
|
||||
WeightsMapper,
|
||||
init_vllm_registered_model,
|
||||
maybe_prefix,
|
||||
)
|
||||
from vllm.model_executor.models.vision import is_vit_use_data_parallel
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.inputs import NestedTensors
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.transformers_utils.configs.kimi_k3 import KimiK3Config
|
||||
|
||||
from ..common.mm_preprocess import (
|
||||
KimiK3DummyInputsBuilder,
|
||||
KimiK3MultiModalProcessor,
|
||||
KimiK3ProcessingInfo,
|
||||
)
|
||||
from .linear import KimiLinearForCausalLM
|
||||
|
||||
|
||||
@MULTIMODAL_REGISTRY.register_processor(
|
||||
KimiK3MultiModalProcessor,
|
||||
info=KimiK3ProcessingInfo,
|
||||
dummy_inputs=KimiK3DummyInputsBuilder,
|
||||
)
|
||||
class KimiK3ForConditionalGeneration(
|
||||
nn.Module,
|
||||
SupportsMultiModal,
|
||||
SupportsPP,
|
||||
SupportsQuant,
|
||||
SupportsEagle3,
|
||||
HasInnerState,
|
||||
IsHybrid,
|
||||
):
|
||||
"""Kimi-K3 model with Kimi-K2.5 vision and KimiLinear text."""
|
||||
|
||||
supports_encoder_tp_data = True
|
||||
|
||||
hf_to_vllm_mapper = WeightsMapper(
|
||||
orig_to_new_prefix={
|
||||
"language_model.layers.": "language_model.model.layers.",
|
||||
"mm_projector.proj.0": "mm_projector.linear_1",
|
||||
"mm_projector.proj.2": "mm_projector.linear_2",
|
||||
}
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def get_placeholder_str(cls, modality: str, i: int) -> str | None:
|
||||
if modality == "image":
|
||||
return "<|kimi_image_placeholder|>"
|
||||
raise ValueError(f"Unsupported modality: {modality}")
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vllm_config: VllmConfig,
|
||||
prefix: str = "",
|
||||
) -> None:
|
||||
super().__init__()
|
||||
model_config = vllm_config.model_config
|
||||
config: KimiK3Config = model_config.hf_config
|
||||
self.config = config
|
||||
quant_config = vllm_config.quant_config
|
||||
|
||||
multimodal_config = model_config.multimodal_config
|
||||
assert multimodal_config is not None
|
||||
self.use_data_parallel = is_vit_use_data_parallel(
|
||||
config.vision_config.num_attention_heads
|
||||
)
|
||||
self.hidden_size = config.text_config.hidden_size
|
||||
self.device = current_platform.current_device()
|
||||
|
||||
with self._mark_tower_model(vllm_config, "image"):
|
||||
self.vision_tower = MoonViT3dPretrainedModel(
|
||||
config.vision_config,
|
||||
quant_config=self._maybe_ignore_quant_config(quant_config),
|
||||
prefix=maybe_prefix(prefix, "vision_tower"),
|
||||
)
|
||||
if self._maybe_ignore_quant_config(quant_config) is not None:
|
||||
self.vision_tower = self.vision_tower.to(device=self.device)
|
||||
else:
|
||||
self.vision_tower = self.vision_tower.to(
|
||||
device=self.device, dtype=model_config.dtype
|
||||
)
|
||||
|
||||
self.mm_projector = KimiK25MultiModalProjector(
|
||||
config=config.vision_config,
|
||||
use_data_parallel=self.use_data_parallel,
|
||||
quant_config=self._maybe_ignore_quant_config(quant_config),
|
||||
prefix=maybe_prefix(prefix, "mm_projector"),
|
||||
)
|
||||
self.mm_projector = self.mm_projector.to(
|
||||
device=self.device, dtype=model_config.dtype
|
||||
)
|
||||
|
||||
self.quant_config = quant_config
|
||||
with self._mark_language_model(vllm_config):
|
||||
self.language_model = init_vllm_registered_model(
|
||||
vllm_config=vllm_config,
|
||||
hf_config=config.text_config,
|
||||
prefix=maybe_prefix(prefix, "language_model"),
|
||||
architectures=["KimiLinearForCausalLM"],
|
||||
)
|
||||
self.make_empty_intermediate_tensors = ( # type: ignore[method-assign]
|
||||
self.language_model.make_empty_intermediate_tensors
|
||||
)
|
||||
self.media_placeholder: int = self.config.media_placeholder_token_id
|
||||
|
||||
def _maybe_ignore_quant_config(
|
||||
self, quant_config: QuantizationConfig | None
|
||||
) -> QuantizationConfig | None:
|
||||
if isinstance(quant_config, compressed_tensors.CompressedTensorsConfig):
|
||||
return None
|
||||
return quant_config
|
||||
|
||||
def _parse_and_validate_media_input(
|
||||
self, **kwargs: object
|
||||
) -> KimiK25MediaPixelInputs | None:
|
||||
pixel_values = kwargs.pop("pixel_values", None)
|
||||
grid_thws = kwargs.pop("grid_thws", None)
|
||||
if pixel_values is None:
|
||||
return None
|
||||
|
||||
if isinstance(pixel_values, list):
|
||||
pixel_values = torch.cat(cast(list[torch.Tensor], pixel_values), dim=0)
|
||||
if not isinstance(pixel_values, torch.Tensor):
|
||||
raise TypeError(
|
||||
"pixel_values must be a tensor or a list of tensors, "
|
||||
f"got {type(pixel_values)}"
|
||||
)
|
||||
|
||||
if len(pixel_values.shape) == 5 or len(pixel_values.shape) == 3:
|
||||
pixel_values = pixel_values.reshape(
|
||||
pixel_values.shape[0] * pixel_values.shape[1], *pixel_values.shape[2:]
|
||||
)
|
||||
|
||||
target_dtype = next(self.vision_tower.parameters()).dtype
|
||||
pixel_values = pixel_values.to(target_dtype)
|
||||
assert isinstance(grid_thws, torch.Tensor), (
|
||||
f"expect grid_thws to be a tensor, got {type(grid_thws)}"
|
||||
)
|
||||
grid_thws = grid_thws.reshape(-1, grid_thws.shape[-1])
|
||||
assert grid_thws.ndim == 2 and grid_thws.size(1) == 3, (
|
||||
f"unexpected shape for grid_thws: {grid_thws.shape}"
|
||||
)
|
||||
|
||||
return KimiK25MediaPixelInputs(
|
||||
type="pixel_values",
|
||||
pixel_values=pixel_values,
|
||||
grid_thws=grid_thws,
|
||||
)
|
||||
|
||||
def _process_media_input(
|
||||
self, media_input: KimiK25MediaPixelInputs
|
||||
) -> list[torch.Tensor]:
|
||||
media_features = vision_tower_forward(
|
||||
self.vision_tower,
|
||||
media_input["pixel_values"],
|
||||
media_input["grid_thws"],
|
||||
mm_projector=self.mm_projector,
|
||||
use_data_parallel=self.use_data_parallel,
|
||||
)
|
||||
return media_features
|
||||
|
||||
def embed_multimodal(self, **kwargs: object) -> NestedTensors | None:
|
||||
media_input = self._parse_and_validate_media_input(**kwargs)
|
||||
if media_input is None:
|
||||
return None
|
||||
return self._process_media_input(media_input)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
intermediate_tensors: IntermediateTensors | None = None,
|
||||
inputs_embeds: torch.Tensor | None = None,
|
||||
**kwargs: object,
|
||||
) -> IntermediateTensors:
|
||||
if intermediate_tensors is not None:
|
||||
inputs_embeds = None
|
||||
hidden_states = self.language_model(
|
||||
input_ids=input_ids,
|
||||
positions=positions,
|
||||
intermediate_tensors=intermediate_tensors,
|
||||
inputs_embeds=inputs_embeds,
|
||||
)
|
||||
return hidden_states
|
||||
|
||||
def compute_logits(self, hidden_states: torch.Tensor, **kwargs) -> torch.Tensor:
|
||||
return self.language_model.compute_logits(hidden_states)
|
||||
|
||||
def copy_inputs_before_cuda_graphs(self, input_buffers, **kwargs):
|
||||
return self.language_model.mamba_cache.copy_inputs_before_cuda_graphs(
|
||||
input_buffers, **kwargs
|
||||
)
|
||||
|
||||
def get_seqlen_agnostic_capture_inputs(self, batch_size: int):
|
||||
return self.language_model.mamba_cache.get_seqlen_agnostic_capture_inputs(
|
||||
batch_size
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def get_mamba_state_dtype_from_config(cls, vllm_config: VllmConfig):
|
||||
text_config = vllm_config.model_config.hf_config.text_config
|
||||
temp_vllm_config = vllm_config.with_hf_config(text_config)
|
||||
return KimiLinearForCausalLM.get_mamba_state_dtype_from_config(temp_vllm_config)
|
||||
|
||||
@classmethod
|
||||
def get_mamba_state_shape_from_config(cls, vllm_config: VllmConfig):
|
||||
text_config = vllm_config.model_config.hf_config.text_config
|
||||
temp_vllm_config = vllm_config.with_hf_config(text_config)
|
||||
return KimiLinearForCausalLM.get_mamba_state_shape_from_config(temp_vllm_config)
|
||||
|
||||
@classmethod
|
||||
def get_mamba_state_copy_func(cls):
|
||||
return KimiLinearForCausalLM.get_mamba_state_copy_func()
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]):
|
||||
loader = AutoWeightsLoader(self)
|
||||
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
|
||||
@@ -0,0 +1,403 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Inference-only Kimi-K3 Multi-Token-Prediction (MTP) draft model."""
|
||||
|
||||
import copy
|
||||
from collections.abc import Iterable
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
fused_moe_make_expert_params_mapping,
|
||||
)
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
ParallelLMHead,
|
||||
VocabParallelEmbedding,
|
||||
)
|
||||
from vllm.model_executor.model_loader.weight_utils import (
|
||||
default_weight_loader,
|
||||
maybe_remap_kv_scale_name,
|
||||
)
|
||||
from vllm.model_executor.models.utils import get_pp_missing_layer_names, maybe_prefix
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.transformers_utils.configs.kimi_linear import KimiLinearConfig
|
||||
|
||||
from ..common.mtp import fused_mtp_input
|
||||
from .linear import KimiDecoderLayer, get_spec_layer_idx_from_weight_name
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class SharedHead(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
config: KimiLinearConfig,
|
||||
prefix: str,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
self.head = ParallelLMHead(
|
||||
config.vocab_size,
|
||||
config.hidden_size,
|
||||
quant_config=quant_config,
|
||||
prefix=maybe_prefix(prefix, "head"),
|
||||
)
|
||||
|
||||
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
||||
return self.norm(hidden_states)
|
||||
|
||||
|
||||
class KimiK3MultiTokenPredictorLayer(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
config: KimiLinearConfig,
|
||||
vllm_config: VllmConfig,
|
||||
prefix: str,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.config = config
|
||||
quant_config = vllm_config.quant_config
|
||||
|
||||
self.enorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
self.hnorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
self.eh_proj = nn.Linear(config.hidden_size * 2, config.hidden_size, bias=False)
|
||||
|
||||
self.shared_head = SharedHead(
|
||||
config=config, prefix=prefix, quant_config=quant_config
|
||||
)
|
||||
# The MTP block is a standard KimiDecoderLayer, but it must NOT use the
|
||||
# attn-residual (block-residual) scheme even when the base model does:
|
||||
# the draft starts from the target's hidden state, without the main
|
||||
# model's accumulated per-block residual tensor. We disable it by
|
||||
# shallow-copying the config with ``attn_res_block_size=None``.
|
||||
block_config = copy.copy(config)
|
||||
block_config.attn_res_block_size = None
|
||||
# NOTE: the prefix must end in the numeric spec-layer index so that
|
||||
# KimiDecoderLayer can parse ``layer_idx`` and pick MLA (full attn).
|
||||
self.mtp_block = KimiDecoderLayer(block_config, vllm_config, prefix=prefix)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
previous_hidden_states: torch.Tensor,
|
||||
inputs_embeds: torch.Tensor | None = None,
|
||||
spec_step_index: int = 0,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
assert inputs_embeds is not None
|
||||
hidden_states = self.eh_proj(
|
||||
fused_mtp_input(
|
||||
positions,
|
||||
inputs_embeds,
|
||||
previous_hidden_states,
|
||||
self.enorm.weight,
|
||||
self.hnorm.weight,
|
||||
self.enorm.variance_epsilon,
|
||||
)
|
||||
)
|
||||
|
||||
hidden_states, residual = self.mtp_block(
|
||||
positions=positions,
|
||||
hidden_states=hidden_states,
|
||||
residual=None,
|
||||
)
|
||||
# Produce the normalized logits input and the pre-norm recurrent state
|
||||
# in one fused add-RMSNorm launch.
|
||||
logits_hidden_states, hidden_states = self.shared_head.norm(
|
||||
hidden_states, residual
|
||||
)
|
||||
return logits_hidden_states, hidden_states
|
||||
|
||||
|
||||
class KimiK3MultiTokenPredictor(nn.Module):
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
super().__init__()
|
||||
config: KimiLinearConfig = vllm_config.model_config.hf_text_config
|
||||
self.config = config
|
||||
self.mtp_start_layer_idx = config.num_hidden_layers
|
||||
self.num_mtp_layers = config.num_nextn_predict_layers
|
||||
|
||||
self.layers = torch.nn.ModuleDict(
|
||||
{
|
||||
str(idx): KimiK3MultiTokenPredictorLayer(
|
||||
config, vllm_config, f"{prefix}.layers.{idx}"
|
||||
)
|
||||
for idx in range(
|
||||
self.mtp_start_layer_idx,
|
||||
self.mtp_start_layer_idx + self.num_mtp_layers,
|
||||
)
|
||||
}
|
||||
)
|
||||
self.embed_tokens = VocabParallelEmbedding(
|
||||
config.vocab_size,
|
||||
config.hidden_size,
|
||||
prefix=maybe_prefix(prefix, "embed_tokens"),
|
||||
)
|
||||
self.logits_processor = LogitsProcessor(config.vocab_size)
|
||||
|
||||
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
|
||||
return self.embed_tokens(input_ids)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
previous_hidden_states: torch.Tensor,
|
||||
inputs_embeds: torch.Tensor | None = None,
|
||||
spec_step_idx: int = 0,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
if inputs_embeds is None:
|
||||
inputs_embeds = self.embed_tokens(input_ids)
|
||||
current_step_idx = spec_step_idx % self.num_mtp_layers
|
||||
return self.layers[str(self.mtp_start_layer_idx + current_step_idx)](
|
||||
input_ids,
|
||||
positions,
|
||||
previous_hidden_states,
|
||||
inputs_embeds,
|
||||
current_step_idx,
|
||||
)
|
||||
|
||||
def compute_logits(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
spec_step_idx: int = 0,
|
||||
) -> torch.Tensor:
|
||||
current_step_idx = spec_step_idx % self.num_mtp_layers
|
||||
mtp_layer = self.layers[str(self.mtp_start_layer_idx + current_step_idx)]
|
||||
logits = self.logits_processor(mtp_layer.shared_head.head, hidden_states)
|
||||
return logits
|
||||
|
||||
|
||||
class KimiK3MTP(nn.Module):
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
super().__init__()
|
||||
self.config = vllm_config.model_config.hf_text_config
|
||||
self.quant_config = vllm_config.quant_config
|
||||
self.model = KimiK3MultiTokenPredictor(
|
||||
vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model")
|
||||
)
|
||||
|
||||
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
|
||||
return self.model.embed_input_ids(input_ids)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.Tensor | None,
|
||||
positions: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
intermediate_tensors: IntermediateTensors | None = None,
|
||||
inputs_embeds: torch.Tensor | None = None,
|
||||
spec_step_idx: int = 0,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
return self.model(
|
||||
input_ids,
|
||||
positions,
|
||||
hidden_states,
|
||||
inputs_embeds,
|
||||
spec_step_idx,
|
||||
)
|
||||
|
||||
def compute_logits(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
spec_step_idx: int = 0,
|
||||
) -> torch.Tensor | None:
|
||||
return self.model.compute_logits(hidden_states, spec_step_idx)
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
# Mirror KimiLinearForCausalLM.load_weights naming: leading-dot shard
|
||||
# names, q_lora-conditional fused QKV, and w1/w2/w3 expert weights.
|
||||
kda_config = self.config.linear_attn_config
|
||||
use_full_rank_gate = bool(
|
||||
kda_config and kda_config.get("use_full_rank_gate", False)
|
||||
)
|
||||
beta_shard_id = 5 if use_full_rank_gate else 3
|
||||
stacked_params_mapping = [
|
||||
# (param_name, shard_name, shard_id)
|
||||
(".in_proj_qkvgfab", ".q_proj", 0),
|
||||
(".in_proj_qkvgfab", ".k_proj", 1),
|
||||
(".in_proj_qkvgfab", ".v_proj", 2),
|
||||
(".in_proj_qkvgfab", ".b_proj", beta_shard_id),
|
||||
(".in_proj_qkvgfab", ".f_a_proj", 4),
|
||||
(".conv1d", ".q_conv1d", 0),
|
||||
(".conv1d", ".k_conv1d", 1),
|
||||
(".conv1d", ".v_conv1d", 2),
|
||||
(".gate_up_proj", ".gate_proj", 0),
|
||||
(".gate_up_proj", ".up_proj", 1),
|
||||
]
|
||||
if use_full_rank_gate:
|
||||
stacked_params_mapping.append((".in_proj_qkvgfab", ".g_proj", 3))
|
||||
if getattr(self.config, "q_lora_rank", None) is not None:
|
||||
stacked_params_mapping += [
|
||||
(".fused_qkv_a_proj", ".q_a_proj", 0),
|
||||
(".fused_qkv_a_proj", ".kv_a_proj_with_mqa", 1),
|
||||
]
|
||||
|
||||
expert_params_mapping = (
|
||||
fused_moe_make_expert_params_mapping(
|
||||
self,
|
||||
ckpt_gate_proj_name="w1",
|
||||
ckpt_down_proj_name="w2",
|
||||
ckpt_up_proj_name="w3",
|
||||
num_experts=self.config.num_experts,
|
||||
)
|
||||
if self.config.is_moe
|
||||
else []
|
||||
)
|
||||
|
||||
pp_missing_layer_names = get_pp_missing_layer_names(self)
|
||||
params_dict = dict(self.named_parameters())
|
||||
# Under the MXFP4 quant interface the routed experts register unpacked
|
||||
# params (``w13_weight``), while the compressed-tensors checkpoint names
|
||||
# them ``.weight_packed``. Rebind so the expert mapping resolves; scales
|
||||
# already share the ``.weight_scale`` suffix.
|
||||
experts_unpacked = not any(n.endswith("w13_weight_packed") for n in params_dict)
|
||||
loaded_params: set[str] = set()
|
||||
for name, loaded_weight in weights:
|
||||
if "rotary_emb.inv_freq" in name:
|
||||
continue
|
||||
# The multimodal checkpoint prefixes text weights with
|
||||
# ``language_model.``; strip it so names match this draft model's
|
||||
# parameter paths (``model.layers.{i}.``). Non-text weights
|
||||
# (vision_tower, mm_projector, ...) never match a spec layer below.
|
||||
if name.startswith("language_model."):
|
||||
name = name[len("language_model.") :]
|
||||
if experts_unpacked and name.endswith(".weight_packed"):
|
||||
name = name.replace(".weight_packed", ".weight")
|
||||
spec_layer = get_spec_layer_idx_from_weight_name(self.config, name)
|
||||
if spec_layer is None:
|
||||
continue
|
||||
name = self._rewrite_spec_layer_name(spec_layer, name)
|
||||
|
||||
for param_name, weight_name, shard_id in stacked_params_mapping:
|
||||
if weight_name not in name:
|
||||
continue
|
||||
# Routed experts (``.experts.{i}.w1/w2/w3``) are handled by the
|
||||
# expert mapping below; skip them here. Shared experts
|
||||
# (``.shared_experts.``) use gate/up_proj and fall through.
|
||||
if ".experts." in name:
|
||||
continue
|
||||
name_mapped = name.replace(weight_name, param_name)
|
||||
# Only take this mapping if the fused destination actually
|
||||
# exists (e.g. QKV fusion is only present when q_lora is used).
|
||||
if name_mapped not in params_dict:
|
||||
continue
|
||||
if name_mapped in pp_missing_layer_names:
|
||||
continue
|
||||
name = name_mapped
|
||||
param = params_dict[name]
|
||||
weight_loader = param.weight_loader
|
||||
weight_loader(param, loaded_weight, shard_id)
|
||||
break
|
||||
else:
|
||||
for (
|
||||
expert_param_name,
|
||||
expert_weight_name,
|
||||
expert_id,
|
||||
expert_shard_id,
|
||||
) in expert_params_mapping:
|
||||
if expert_weight_name not in name:
|
||||
continue
|
||||
name_mapped = name.replace(expert_weight_name, expert_param_name)
|
||||
if name_mapped in pp_missing_layer_names:
|
||||
continue
|
||||
param = params_dict[name_mapped]
|
||||
weight_loader = param.weight_loader
|
||||
weight_loader(
|
||||
param,
|
||||
loaded_weight,
|
||||
name_mapped,
|
||||
shard_id=expert_shard_id,
|
||||
expert_id=expert_id,
|
||||
)
|
||||
name = name_mapped
|
||||
break
|
||||
else:
|
||||
if name.endswith(".bias") and name not in params_dict:
|
||||
continue
|
||||
remapped_name = maybe_remap_kv_scale_name(name, params_dict)
|
||||
if remapped_name is None:
|
||||
continue
|
||||
name = remapped_name
|
||||
|
||||
# The embedding is shared across MTP layers; only the first
|
||||
# spec layer carries the hoisted (non-".layers") copy.
|
||||
if spec_layer != self.model.mtp_start_layer_idx and (
|
||||
".layers" not in name
|
||||
):
|
||||
continue
|
||||
if name in pp_missing_layer_names:
|
||||
continue
|
||||
# The base model uses an attn-residual scheme whose per-layer
|
||||
# weights (self_attention_res_*, mlp_res_*) are not used by
|
||||
# the draft block; such names have no matching parameter and
|
||||
# are safely skipped.
|
||||
if name not in params_dict:
|
||||
continue
|
||||
|
||||
param = params_dict[name]
|
||||
weight_loader = getattr(
|
||||
param, "weight_loader", default_weight_loader
|
||||
)
|
||||
weight_loader(param, loaded_weight)
|
||||
loaded_params.add(name)
|
||||
|
||||
# Validate that weights were loaded for each expected MTP layer.
|
||||
loaded_layers: set[int] = set()
|
||||
for param_name in loaded_params:
|
||||
spec_layer = get_spec_layer_idx_from_weight_name(self.config, param_name)
|
||||
if spec_layer is not None:
|
||||
loaded_layers.add(spec_layer)
|
||||
for layer_idx in range(
|
||||
self.model.mtp_start_layer_idx,
|
||||
self.model.mtp_start_layer_idx + self.model.num_mtp_layers,
|
||||
):
|
||||
if layer_idx not in loaded_layers:
|
||||
raise ValueError(
|
||||
f"MTP speculative decoding layer {layer_idx} weights "
|
||||
f"missing from checkpoint. The checkpoint may not include "
|
||||
f"the MTP layer weights. Use a checkpoint that includes "
|
||||
f"MTP layer weights, or disable speculative decoding."
|
||||
)
|
||||
|
||||
return loaded_params
|
||||
|
||||
def _rewrite_spec_layer_name(self, spec_layer: int, name: str) -> str:
|
||||
"""Rewrite a checkpoint weight name to this module's parameter path.
|
||||
|
||||
Top-level MTP submodules (enorm/hnorm/eh_proj/shared_head) stay under
|
||||
``model.layers.{spec_layer}.*``; the shared ``embed_tokens`` is hoisted
|
||||
to ``model.*``; everything else is a transformer-block weight and gets
|
||||
``.mtp_block`` inserted.
|
||||
"""
|
||||
spec_layer_weight_names = [
|
||||
"embed_tokens",
|
||||
"enorm",
|
||||
"hnorm",
|
||||
"eh_proj",
|
||||
"shared_head",
|
||||
]
|
||||
shared_weight_names = ["embed_tokens"]
|
||||
spec_layer_weight = False
|
||||
shared_weight = False
|
||||
for weight_name in spec_layer_weight_names:
|
||||
if weight_name in name:
|
||||
spec_layer_weight = True
|
||||
if weight_name in shared_weight_names:
|
||||
shared_weight = True
|
||||
break
|
||||
if not spec_layer_weight:
|
||||
name = name.replace(
|
||||
f"model.layers.{spec_layer}.",
|
||||
f"model.layers.{spec_layer}.mtp_block.",
|
||||
)
|
||||
elif shared_weight:
|
||||
name = name.replace(f"model.layers.{spec_layer}.", "model.")
|
||||
return name
|
||||
@@ -0,0 +1,2 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
@@ -0,0 +1,47 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
#
|
||||
# AMD/ROCm vendored copy of the Kimi-K3 KDA triton kernels.
|
||||
#
|
||||
# Provenance: mirror of vllm/models/kimi_k3/nvidia/ops/third_party/kda (tracker
|
||||
# mke-tracker @ 7adebfcf9; FLA vendored per PRs #39/#86). Split per-vendor so
|
||||
# AMD can carry gfx950-specific kernel changes without touching the NVIDIA copy.
|
||||
#
|
||||
# fla-org/flash-linear-attention#869 (the unmerged ROCm fixes our earlier
|
||||
# amd_fla shim carried against FLA 0.5.0) is covered here by the *newer* vendored
|
||||
# FLA rather than the literal patch:
|
||||
# - transpose-state-layout workaround: N/A (kernels rewritten; no
|
||||
# transpose_state_layout path remains),
|
||||
# - AMD autotune configs: present (is_amd num_warps/num_stages branches),
|
||||
# - OOB-mask correctness fix: present (all tl.load use mask=..., other=0).
|
||||
# Validated on gfx950: no core-dump, gsm8k 94.1%.
|
||||
#
|
||||
# AMD-specific deltas vs the NVIDIA copy: NONE yet (byte-identical). Keep in sync
|
||||
# with the NVIDIA copy on FLA updates; any divergence should be an intentional,
|
||||
# documented gfx950-specific change (a #869-style AMD-only fix).
|
||||
|
||||
from .chunk import (
|
||||
chunk_kda,
|
||||
chunk_kda_fwd,
|
||||
chunk_kda_with_fused_gate,
|
||||
chunk_kda_with_fused_gate_fwd,
|
||||
fused_kda_gate,
|
||||
fused_kda_gate_chunk_cumsum,
|
||||
)
|
||||
from .fused_recurrent import (
|
||||
fused_recurrent_kda,
|
||||
fused_recurrent_kda_fwd,
|
||||
fused_recurrent_kda_packed_decode,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"chunk_kda",
|
||||
"chunk_kda_fwd",
|
||||
"chunk_kda_with_fused_gate",
|
||||
"chunk_kda_with_fused_gate_fwd",
|
||||
"fused_kda_gate",
|
||||
"fused_kda_gate_chunk_cumsum",
|
||||
"fused_recurrent_kda",
|
||||
"fused_recurrent_kda_fwd",
|
||||
"fused_recurrent_kda_packed_decode",
|
||||
]
|
||||
@@ -0,0 +1,935 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# SPDX-FileCopyrightText: Songlin Yang, Yu Zhang, Zhiyuan Li
|
||||
#
|
||||
# This file contains code copied from the flash-linear-attention project.
|
||||
# The original source code was licensed under the MIT license and included
|
||||
# the following copyright notice:
|
||||
# Copyright (c) 2023-2026, Songlin Yang, Yu Zhang, Zhiyuan Li
|
||||
# ruff: noqa: E501
|
||||
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.third_party.flash_linear_attention.ops.chunk_delta_h import (
|
||||
chunk_gated_delta_rule_fwd_h,
|
||||
)
|
||||
from vllm.third_party.flash_linear_attention.ops.cumsum import chunk_local_cumsum
|
||||
from vllm.third_party.flash_linear_attention.ops.index import prepare_chunk_indices
|
||||
from vllm.third_party.flash_linear_attention.ops.l2norm import l2norm_fwd
|
||||
from vllm.third_party.flash_linear_attention.ops.op import exp2, log
|
||||
from vllm.third_party.flash_linear_attention.ops.utils import FLA_CHUNK_SIZE, is_amd
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.utils.math_utils import RCP_LN2, cdiv, next_power_of_2
|
||||
|
||||
from .chunk_intra import chunk_kda_fwd_intra
|
||||
|
||||
BT_LIST_AUTOTUNE = [32, 64, 128]
|
||||
NUM_WARPS_AUTOTUNE = [2, 4, 8, 16] if is_amd else [4, 8, 16, 32]
|
||||
|
||||
|
||||
@triton.heuristics(
|
||||
{
|
||||
"STORE_QG": lambda args: args["qg"] is not None,
|
||||
"STORE_KG": lambda args: args["kg"] is not None,
|
||||
"IS_VARLEN": lambda args: args["cu_seqlens"] is not None,
|
||||
}
|
||||
)
|
||||
@triton.autotune(
|
||||
configs=[
|
||||
triton.Config({}, num_warps=num_warps, num_stages=num_stages)
|
||||
for num_warps in [2, 4, 8]
|
||||
for num_stages in [2, 3, 4]
|
||||
],
|
||||
key=["H", "K", "V", "BT", "BK", "BV", "IS_VARLEN"],
|
||||
)
|
||||
@triton.jit(do_not_specialize=["T"])
|
||||
def recompute_w_u_fwd_kernel(
|
||||
q,
|
||||
k,
|
||||
qg,
|
||||
kg,
|
||||
v,
|
||||
beta,
|
||||
w,
|
||||
u,
|
||||
A,
|
||||
gk,
|
||||
cu_seqlens,
|
||||
chunk_indices,
|
||||
T,
|
||||
H: tl.constexpr,
|
||||
K: tl.constexpr,
|
||||
V: tl.constexpr,
|
||||
BT: tl.constexpr,
|
||||
BK: tl.constexpr,
|
||||
BV: tl.constexpr,
|
||||
STORE_QG: tl.constexpr,
|
||||
STORE_KG: tl.constexpr,
|
||||
IS_VARLEN: tl.constexpr,
|
||||
DOT_PRECISION: tl.constexpr,
|
||||
):
|
||||
i_t, i_bh = tl.program_id(0), tl.program_id(1)
|
||||
i_b, i_h = i_bh // H, i_bh % H
|
||||
if IS_VARLEN:
|
||||
i_n, i_t = (
|
||||
tl.load(chunk_indices + i_t * 2).to(tl.int32),
|
||||
tl.load(chunk_indices + i_t * 2 + 1).to(tl.int32),
|
||||
)
|
||||
bos, eos = (
|
||||
tl.load(cu_seqlens + i_n).to(tl.int32),
|
||||
tl.load(cu_seqlens + i_n + 1).to(tl.int32),
|
||||
)
|
||||
T = eos - bos
|
||||
else:
|
||||
bos, eos = i_b * T, i_b * T + T
|
||||
p_b = tl.make_block_ptr(beta + bos * H + i_h, (T,), (H,), (i_t * BT,), (BT,), (0,))
|
||||
b_b = tl.load(p_b, boundary_check=(0,)).to(tl.float32)
|
||||
|
||||
p_A = tl.make_block_ptr(
|
||||
A + (bos * H + i_h) * BT, (T, BT), (H * BT, 1), (i_t * BT, 0), (BT, BT), (1, 0)
|
||||
)
|
||||
b_A = tl.load(p_A, boundary_check=(0, 1))
|
||||
|
||||
for i_v in range(tl.cdiv(V, BV)):
|
||||
p_v = tl.make_block_ptr(
|
||||
v + (bos * H + i_h) * V,
|
||||
(T, V),
|
||||
(H * V, 1),
|
||||
(i_t * BT, i_v * BV),
|
||||
(BT, BV),
|
||||
(1, 0),
|
||||
)
|
||||
p_u = tl.make_block_ptr(
|
||||
u + (bos * H + i_h) * V,
|
||||
(T, V),
|
||||
(H * V, 1),
|
||||
(i_t * BT, i_v * BV),
|
||||
(BT, BV),
|
||||
(1, 0),
|
||||
)
|
||||
b_v = tl.load(p_v, boundary_check=(0, 1))
|
||||
b_vb = (b_v * b_b[:, None]).to(b_v.dtype)
|
||||
b_u = tl.dot(b_A, b_vb, input_precision=DOT_PRECISION)
|
||||
tl.store(p_u, b_u.to(p_u.dtype.element_ty), boundary_check=(0, 1))
|
||||
|
||||
for i_k in range(tl.cdiv(K, BK)):
|
||||
p_w = tl.make_block_ptr(
|
||||
w + (bos * H + i_h) * K,
|
||||
(T, K),
|
||||
(H * K, 1),
|
||||
(i_t * BT, i_k * BK),
|
||||
(BT, BK),
|
||||
(1, 0),
|
||||
)
|
||||
p_k = tl.make_block_ptr(
|
||||
k + (bos * H + i_h) * K,
|
||||
(T, K),
|
||||
(H * K, 1),
|
||||
(i_t * BT, i_k * BK),
|
||||
(BT, BK),
|
||||
(1, 0),
|
||||
)
|
||||
b_k = tl.load(p_k, boundary_check=(0, 1))
|
||||
b_kb = b_k * b_b[:, None]
|
||||
|
||||
p_gk = tl.make_block_ptr(
|
||||
gk + (bos * H + i_h) * K,
|
||||
(T, K),
|
||||
(H * K, 1),
|
||||
(i_t * BT, i_k * BK),
|
||||
(BT, BK),
|
||||
(1, 0),
|
||||
)
|
||||
b_gk = tl.load(p_gk, boundary_check=(0, 1))
|
||||
b_kb *= exp2(b_gk)
|
||||
if STORE_QG:
|
||||
p_q = tl.make_block_ptr(
|
||||
q + (bos * H + i_h) * K,
|
||||
(T, K),
|
||||
(H * K, 1),
|
||||
(i_t * BT, i_k * BK),
|
||||
(BT, BK),
|
||||
(1, 0),
|
||||
)
|
||||
p_qg = tl.make_block_ptr(
|
||||
qg + (bos * H + i_h) * K,
|
||||
(T, K),
|
||||
(H * K, 1),
|
||||
(i_t * BT, i_k * BK),
|
||||
(BT, BK),
|
||||
(1, 0),
|
||||
)
|
||||
b_q = tl.load(p_q, boundary_check=(0, 1))
|
||||
b_qg = b_q * exp2(b_gk)
|
||||
tl.store(p_qg, b_qg.to(p_qg.dtype.element_ty), boundary_check=(0, 1))
|
||||
if STORE_KG:
|
||||
last_idx = min(i_t * BT + BT, T) - 1
|
||||
|
||||
o_k = i_k * BK + tl.arange(0, BK)
|
||||
m_k = o_k < K
|
||||
b_gn = tl.load(
|
||||
gk + ((bos + last_idx) * H + i_h) * K + o_k, mask=m_k, other=0.0
|
||||
)
|
||||
b_kg = b_k * exp2(b_gn - b_gk)
|
||||
|
||||
p_kg = tl.make_block_ptr(
|
||||
kg + (bos * H + i_h) * K,
|
||||
(T, K),
|
||||
(H * K, 1),
|
||||
(i_t * BT, i_k * BK),
|
||||
(BT, BK),
|
||||
(1, 0),
|
||||
)
|
||||
tl.store(p_kg, b_kg.to(p_kg.dtype.element_ty), boundary_check=(0, 1))
|
||||
|
||||
b_w = tl.dot(b_A, b_kb.to(b_k.dtype))
|
||||
tl.store(p_w, b_w.to(p_w.dtype.element_ty), boundary_check=(0, 1))
|
||||
|
||||
|
||||
def recompute_w_u_fwd(
|
||||
k: torch.Tensor,
|
||||
v: torch.Tensor,
|
||||
beta: torch.Tensor,
|
||||
A: torch.Tensor,
|
||||
q: torch.Tensor | None = None,
|
||||
gk: torch.Tensor | None = None,
|
||||
cu_seqlens: torch.Tensor | None = None,
|
||||
chunk_indices: torch.Tensor | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
B, T, H, K, V = *k.shape, v.shape[-1]
|
||||
BT = A.shape[-1]
|
||||
BK = 64
|
||||
BV = 64
|
||||
|
||||
if chunk_indices is None and cu_seqlens is not None:
|
||||
chunk_indices = prepare_chunk_indices(cu_seqlens, BT)
|
||||
NT = cdiv(T, BT) if cu_seqlens is None else len(chunk_indices)
|
||||
|
||||
w = torch.empty_like(k)
|
||||
u = torch.empty_like(v)
|
||||
kg = torch.empty_like(k) if gk is not None else None
|
||||
recompute_w_u_fwd_kernel[(NT, B * H)](
|
||||
q=q,
|
||||
k=k,
|
||||
qg=None,
|
||||
kg=kg,
|
||||
v=v,
|
||||
beta=beta,
|
||||
w=w,
|
||||
u=u,
|
||||
A=A,
|
||||
gk=gk,
|
||||
cu_seqlens=cu_seqlens,
|
||||
chunk_indices=chunk_indices,
|
||||
T=T,
|
||||
H=H,
|
||||
K=K,
|
||||
V=V,
|
||||
BT=BT,
|
||||
BK=BK,
|
||||
BV=BV,
|
||||
DOT_PRECISION="ieee",
|
||||
)
|
||||
return w, u, None, kg
|
||||
|
||||
|
||||
@triton.heuristics({"IS_VARLEN": lambda args: args["cu_seqlens"] is not None})
|
||||
@triton.autotune(
|
||||
configs=[
|
||||
triton.Config({"BK": BK, "BV": BV}, num_warps=num_warps, num_stages=num_stages)
|
||||
for BK in [32, 64]
|
||||
for BV in [64, 128]
|
||||
for num_warps in [2, 4, 8]
|
||||
for num_stages in [2, 3, 4]
|
||||
],
|
||||
key=["BT"],
|
||||
)
|
||||
@triton.jit(do_not_specialize=["T"])
|
||||
def chunk_gla_fwd_kernel_o(
|
||||
q,
|
||||
v,
|
||||
g,
|
||||
h,
|
||||
o,
|
||||
A,
|
||||
cu_seqlens,
|
||||
chunk_indices,
|
||||
scale,
|
||||
T,
|
||||
H: tl.constexpr,
|
||||
K: tl.constexpr,
|
||||
V: tl.constexpr,
|
||||
BT: tl.constexpr,
|
||||
BK: tl.constexpr,
|
||||
BV: tl.constexpr,
|
||||
IS_VARLEN: tl.constexpr,
|
||||
):
|
||||
i_v, i_t, i_bh = tl.program_id(0), tl.program_id(1), tl.program_id(2)
|
||||
i_b, i_h = i_bh // H, i_bh % H
|
||||
if IS_VARLEN:
|
||||
i_tg = i_t
|
||||
i_n, i_t = (
|
||||
tl.load(chunk_indices + i_t * 2).to(tl.int32),
|
||||
tl.load(chunk_indices + i_t * 2 + 1).to(tl.int32),
|
||||
)
|
||||
bos, eos = (
|
||||
tl.load(cu_seqlens + i_n).to(tl.int32),
|
||||
tl.load(cu_seqlens + i_n + 1).to(tl.int32),
|
||||
)
|
||||
T = eos - bos
|
||||
NT = tl.cdiv(T, BT)
|
||||
else:
|
||||
NT = tl.cdiv(T, BT)
|
||||
i_tg = i_b * NT + i_t
|
||||
bos, eos = i_b * T, i_b * T + T
|
||||
|
||||
m_s = tl.arange(0, BT)[:, None] >= tl.arange(0, BT)[None, :]
|
||||
|
||||
b_o = tl.zeros([BT, BV], dtype=tl.float32)
|
||||
for i_k in range(tl.cdiv(K, BK)):
|
||||
p_q = tl.make_block_ptr(
|
||||
q + (bos * H + i_h) * K,
|
||||
(T, K),
|
||||
(H * K, 1),
|
||||
(i_t * BT, i_k * BK),
|
||||
(BT, BK),
|
||||
(1, 0),
|
||||
)
|
||||
p_g = tl.make_block_ptr(
|
||||
g + (bos * H + i_h) * K,
|
||||
(T, K),
|
||||
(H * K, 1),
|
||||
(i_t * BT, i_k * BK),
|
||||
(BT, BK),
|
||||
(1, 0),
|
||||
)
|
||||
p_h = tl.make_block_ptr(
|
||||
h + (i_tg * H + i_h) * K * V,
|
||||
(V, K),
|
||||
(K, 1),
|
||||
(i_v * BV, i_k * BK),
|
||||
(BV, BK),
|
||||
(1, 0),
|
||||
)
|
||||
|
||||
# [BT, BK]
|
||||
b_q = tl.load(p_q, boundary_check=(0, 1))
|
||||
b_q = (b_q * scale).to(b_q.dtype)
|
||||
# [BT, BK]
|
||||
b_g = tl.load(p_g, boundary_check=(0, 1))
|
||||
# [BT, BK]
|
||||
b_qg = (b_q * exp2(b_g)).to(b_q.dtype)
|
||||
# [BV, BK]
|
||||
b_h = tl.load(p_h, boundary_check=(0, 1))
|
||||
# [BT, BV]
|
||||
if i_k >= 0:
|
||||
b_o += tl.dot(b_qg, tl.trans(b_h).to(b_qg.dtype))
|
||||
p_v = tl.make_block_ptr(
|
||||
v + (bos * H + i_h) * V,
|
||||
(T, V),
|
||||
(H * V, 1),
|
||||
(i_t * BT, i_v * BV),
|
||||
(BT, BV),
|
||||
(1, 0),
|
||||
)
|
||||
p_o = tl.make_block_ptr(
|
||||
o + (bos * H + i_h) * V,
|
||||
(T, V),
|
||||
(H * V, 1),
|
||||
(i_t * BT, i_v * BV),
|
||||
(BT, BV),
|
||||
(1, 0),
|
||||
)
|
||||
p_A = tl.make_block_ptr(
|
||||
A + (bos * H + i_h) * BT, (T, BT), (H * BT, 1), (i_t * BT, 0), (BT, BT), (1, 0)
|
||||
)
|
||||
# [BT, BV]
|
||||
b_v = tl.load(p_v, boundary_check=(0, 1))
|
||||
# [BT, BT]
|
||||
b_A = tl.load(p_A, boundary_check=(0, 1))
|
||||
b_A = tl.where(m_s, b_A, 0.0).to(b_v.dtype)
|
||||
b_o += tl.dot(b_A, b_v, allow_tf32=False)
|
||||
tl.store(p_o, b_o.to(p_o.dtype.element_ty), boundary_check=(0, 1))
|
||||
|
||||
|
||||
def chunk_gla_fwd_o_gk(
|
||||
q: torch.Tensor,
|
||||
v: torch.Tensor,
|
||||
g: torch.Tensor,
|
||||
A: torch.Tensor,
|
||||
h: torch.Tensor,
|
||||
o: torch.Tensor,
|
||||
scale: float,
|
||||
cu_seqlens: torch.Tensor | None = None,
|
||||
chunk_indices: torch.Tensor | None = None,
|
||||
chunk_size: int = FLA_CHUNK_SIZE,
|
||||
):
|
||||
B, T, H, K, V = *q.shape, v.shape[-1]
|
||||
BT = chunk_size
|
||||
|
||||
if chunk_indices is None and cu_seqlens is not None:
|
||||
chunk_indices = prepare_chunk_indices(cu_seqlens, chunk_size)
|
||||
NT = cdiv(T, BT) if cu_seqlens is None else len(chunk_indices)
|
||||
|
||||
def grid(meta):
|
||||
return (cdiv(V, meta["BV"]), NT, B * H)
|
||||
|
||||
chunk_gla_fwd_kernel_o[grid](
|
||||
q=q,
|
||||
v=v,
|
||||
g=g,
|
||||
h=h,
|
||||
o=o,
|
||||
A=A,
|
||||
cu_seqlens=cu_seqlens,
|
||||
chunk_indices=chunk_indices,
|
||||
scale=scale,
|
||||
T=T,
|
||||
H=H,
|
||||
K=K,
|
||||
V=V,
|
||||
BT=BT,
|
||||
)
|
||||
return o
|
||||
|
||||
|
||||
@triton.heuristics(
|
||||
{
|
||||
"HAS_BIAS": lambda args: args["g_bias"] is not None,
|
||||
"IS_VARLEN": lambda args: args["cu_seqlens"] is not None,
|
||||
}
|
||||
)
|
||||
@triton.autotune(
|
||||
configs=[
|
||||
triton.Config({"BS": BS}, num_warps=num_warps)
|
||||
for BS in [32, 64]
|
||||
for num_warps in [2, 4, 8]
|
||||
],
|
||||
key=["H", "S", "BT", "IS_VARLEN"],
|
||||
)
|
||||
@triton.jit(do_not_specialize=["T"])
|
||||
def kda_gate_chunk_cumsum_vector_kernel(
|
||||
s,
|
||||
raw_beta,
|
||||
A_log,
|
||||
g_bias,
|
||||
o,
|
||||
beta_out,
|
||||
cu_seqlens,
|
||||
chunk_indices,
|
||||
cumsum_scale,
|
||||
lower_bound,
|
||||
beta,
|
||||
threshold,
|
||||
T,
|
||||
stride_beta_batch,
|
||||
stride_beta_token,
|
||||
stride_beta_head,
|
||||
H: tl.constexpr,
|
||||
S: tl.constexpr,
|
||||
BT: tl.constexpr,
|
||||
BS: tl.constexpr,
|
||||
HAS_BIAS: tl.constexpr,
|
||||
IS_VARLEN: tl.constexpr,
|
||||
USE_LOWER_BOUND: tl.constexpr,
|
||||
):
|
||||
i_s, i_t, i_bh = tl.program_id(0), tl.program_id(1), tl.program_id(2)
|
||||
i_b, i_h = i_bh // H, i_bh % H
|
||||
if IS_VARLEN:
|
||||
i_n, i_t = (
|
||||
tl.load(chunk_indices + i_t * 2).to(tl.int32),
|
||||
tl.load(chunk_indices + i_t * 2 + 1).to(tl.int32),
|
||||
)
|
||||
bos, eos = (
|
||||
tl.load(cu_seqlens + i_n).to(tl.int32),
|
||||
tl.load(cu_seqlens + i_n + 1).to(tl.int32),
|
||||
)
|
||||
T = eos - bos
|
||||
else:
|
||||
bos = i_b * T
|
||||
|
||||
if i_s == 0:
|
||||
o_beta_t = tl.arange(0, BT)
|
||||
m_beta = i_t * BT + o_beta_t < T
|
||||
if IS_VARLEN:
|
||||
p_beta = (
|
||||
raw_beta
|
||||
+ (bos + i_t * BT + o_beta_t) * stride_beta_token
|
||||
+ i_h * stride_beta_head
|
||||
)
|
||||
else:
|
||||
p_beta = (
|
||||
raw_beta
|
||||
+ i_b * stride_beta_batch
|
||||
+ (i_t * BT + o_beta_t) * stride_beta_token
|
||||
+ i_h * stride_beta_head
|
||||
)
|
||||
b_beta = tl.load(p_beta, mask=m_beta, other=0.0).to(tl.float32)
|
||||
p_beta_out = beta_out + (bos + i_t * BT + o_beta_t) * H + i_h
|
||||
tl.store(p_beta_out, tl.sigmoid(b_beta), mask=m_beta)
|
||||
return
|
||||
|
||||
i_s -= 1
|
||||
|
||||
p_s = tl.make_block_ptr(
|
||||
s + (bos * H + i_h) * S,
|
||||
(T, S),
|
||||
(H * S, 1),
|
||||
(i_t * BT, i_s * BS),
|
||||
(BT, BS),
|
||||
(1, 0),
|
||||
)
|
||||
p_o = tl.make_block_ptr(
|
||||
o + (bos * H + i_h) * S,
|
||||
(T, S),
|
||||
(H * S, 1),
|
||||
(i_t * BT, i_s * BS),
|
||||
(BT, BS),
|
||||
(1, 0),
|
||||
)
|
||||
|
||||
b_s = tl.load(p_s, boundary_check=(0, 1)).to(tl.float32)
|
||||
if HAS_BIAS:
|
||||
p_bias = tl.make_block_ptr(
|
||||
g_bias + i_h * S,
|
||||
(S,),
|
||||
(1,),
|
||||
(i_s * BS,),
|
||||
(BS,),
|
||||
(0,),
|
||||
)
|
||||
b_bias = tl.load(p_bias, boundary_check=(0,)).to(tl.float32)
|
||||
b_s += b_bias[None, :]
|
||||
|
||||
b_a = tl.exp(tl.load(A_log + i_h).to(tl.float32))
|
||||
if USE_LOWER_BOUND:
|
||||
b_gate = lower_bound * tl.sigmoid(b_a * b_s)
|
||||
else:
|
||||
b_g_scaled = b_s * beta
|
||||
b_softplus = tl.where(
|
||||
b_g_scaled > threshold,
|
||||
b_s,
|
||||
(1.0 / beta) * log(1.0 + tl.exp(b_g_scaled)),
|
||||
)
|
||||
b_gate = -b_a * b_softplus
|
||||
|
||||
# Boundary loads return zero, but bias and gate activation can make padded
|
||||
# rows nonzero. Padding trails valid rows, so it only affects masked stores.
|
||||
b_o = tl.cumsum(b_gate, axis=0) * cumsum_scale
|
||||
tl.store(p_o, b_o.to(p_o.dtype.element_ty), boundary_check=(0, 1))
|
||||
|
||||
|
||||
def fused_kda_gate_chunk_cumsum(
|
||||
raw_g: torch.Tensor,
|
||||
raw_beta: torch.Tensor,
|
||||
A_log: torch.Tensor,
|
||||
g_bias: torch.Tensor | None = None,
|
||||
beta: float = 1.0,
|
||||
threshold: float = 20.0,
|
||||
lower_bound: float | None = None,
|
||||
cu_seqlens: torch.Tensor | None = None,
|
||||
chunk_indices: torch.Tensor | None = None,
|
||||
chunk_size: int = FLA_CHUNK_SIZE,
|
||||
output_dtype: torch.dtype | None = torch.float,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
if cu_seqlens is not None:
|
||||
assert raw_g.shape[0] == 1, (
|
||||
"Only batch size 1 is supported when cu_seqlens are provided"
|
||||
)
|
||||
B, T, H, D = raw_g.shape
|
||||
if raw_beta.shape != (B, T, H):
|
||||
raise ValueError(f"Expected raw_beta shape {(B, T, H)}, got {raw_beta.shape}")
|
||||
if chunk_indices is None and cu_seqlens is not None:
|
||||
chunk_indices = prepare_chunk_indices(cu_seqlens, chunk_size)
|
||||
NT = cdiv(T, chunk_size) if cu_seqlens is None else len(chunk_indices)
|
||||
|
||||
A_log = A_log.reshape(-1)
|
||||
if g_bias is not None:
|
||||
g_bias = g_bias.reshape(-1)
|
||||
y = torch.empty_like(raw_g, dtype=output_dtype or raw_g.dtype)
|
||||
beta_out = torch.empty(raw_beta.shape, device=raw_beta.device, dtype=torch.float32)
|
||||
|
||||
def grid(meta):
|
||||
# For each (chunk, head), program 0 computes beta without extending a
|
||||
# gate tile's critical path. The remaining programs cover the gate dim.
|
||||
return (cdiv(meta["S"], meta["BS"]) + 1, NT, B * H)
|
||||
|
||||
kda_gate_chunk_cumsum_vector_kernel[grid](
|
||||
s=raw_g,
|
||||
raw_beta=raw_beta,
|
||||
A_log=A_log,
|
||||
g_bias=g_bias,
|
||||
o=y,
|
||||
beta_out=beta_out,
|
||||
cu_seqlens=cu_seqlens,
|
||||
chunk_indices=chunk_indices,
|
||||
# RCP_LN2 folds in the natural-log -> log2 conversion so downstream
|
||||
# exp2-based kernels reproduce exp(g). Keep this in sync with the
|
||||
# `use_exp2=True` path in `_chunk_kda_fwd_with_cumulative_g`.
|
||||
cumsum_scale=RCP_LN2,
|
||||
lower_bound=lower_bound or 0.0,
|
||||
beta=beta,
|
||||
threshold=threshold,
|
||||
T=T,
|
||||
stride_beta_batch=raw_beta.stride(0),
|
||||
stride_beta_token=raw_beta.stride(1),
|
||||
stride_beta_head=raw_beta.stride(2),
|
||||
H=H,
|
||||
S=D,
|
||||
BT=chunk_size,
|
||||
USE_LOWER_BOUND=lower_bound is not None,
|
||||
)
|
||||
return y, beta_out
|
||||
|
||||
|
||||
def _chunk_kda_fwd_with_cumulative_g(
|
||||
q: torch.Tensor,
|
||||
k: torch.Tensor,
|
||||
v: torch.Tensor,
|
||||
g: torch.Tensor,
|
||||
beta: torch.Tensor,
|
||||
scale: float,
|
||||
initial_state: torch.Tensor,
|
||||
output_final_state: bool,
|
||||
cu_seqlens: torch.Tensor | None = None,
|
||||
chunk_indices: torch.Tensor | None = None,
|
||||
chunk_size: int = FLA_CHUNK_SIZE,
|
||||
safe_gate: bool = False,
|
||||
):
|
||||
# `g` must already be chunk-local cumulatively-summed AND scaled by
|
||||
# RCP_LN2 (so the downstream exp2-based kernels reproduce exp(g)).
|
||||
# Use `chunk_kda_fwd` or `chunk_kda_with_fused_gate_fwd` instead of
|
||||
# calling this helper directly unless that invariant is upheld.
|
||||
Aqk, A = chunk_kda_fwd_intra(
|
||||
q=q,
|
||||
k=k,
|
||||
gk=g,
|
||||
beta=beta,
|
||||
scale=scale,
|
||||
cu_seqlens=cu_seqlens,
|
||||
chunk_indices=chunk_indices,
|
||||
chunk_size=chunk_size,
|
||||
safe_gate=safe_gate,
|
||||
)
|
||||
w, u, _, kg = recompute_w_u_fwd(
|
||||
k=k,
|
||||
v=v,
|
||||
beta=beta,
|
||||
A=A,
|
||||
gk=g,
|
||||
cu_seqlens=cu_seqlens,
|
||||
chunk_indices=chunk_indices,
|
||||
)
|
||||
del A
|
||||
h, v_new, final_state = chunk_gated_delta_rule_fwd_h(
|
||||
k=kg,
|
||||
w=w,
|
||||
u=u,
|
||||
gk=g,
|
||||
initial_state=initial_state,
|
||||
output_final_state=output_final_state,
|
||||
cu_seqlens=cu_seqlens,
|
||||
chunk_indices=chunk_indices,
|
||||
use_exp2=True,
|
||||
)
|
||||
del w, u, kg
|
||||
o = chunk_gla_fwd_o_gk(
|
||||
q=q,
|
||||
v=v_new,
|
||||
g=g,
|
||||
A=Aqk,
|
||||
h=h,
|
||||
o=v,
|
||||
scale=scale,
|
||||
cu_seqlens=cu_seqlens,
|
||||
chunk_indices=chunk_indices,
|
||||
chunk_size=chunk_size,
|
||||
)
|
||||
del Aqk, v_new, h
|
||||
return o, final_state
|
||||
|
||||
|
||||
def chunk_kda_fwd(
|
||||
q: torch.Tensor,
|
||||
k: torch.Tensor,
|
||||
v: torch.Tensor,
|
||||
g: torch.Tensor,
|
||||
beta: torch.Tensor,
|
||||
scale: float,
|
||||
initial_state: torch.Tensor,
|
||||
output_final_state: bool,
|
||||
cu_seqlens: torch.Tensor | None = None,
|
||||
):
|
||||
chunk_size = FLA_CHUNK_SIZE
|
||||
chunk_indices = (
|
||||
prepare_chunk_indices(cu_seqlens, chunk_size)
|
||||
if cu_seqlens is not None
|
||||
else None
|
||||
)
|
||||
g = chunk_local_cumsum(
|
||||
g,
|
||||
chunk_size=chunk_size,
|
||||
cu_seqlens=cu_seqlens,
|
||||
chunk_indices=chunk_indices,
|
||||
)
|
||||
# KDA evaluates cumulative gate decays with exp2. Convert from natural-log
|
||||
# space so exp(x) is preserved as exp2(x / ln(2)).
|
||||
g = g * RCP_LN2
|
||||
return _chunk_kda_fwd_with_cumulative_g(
|
||||
q=q,
|
||||
k=k,
|
||||
v=v,
|
||||
g=g,
|
||||
beta=beta,
|
||||
scale=scale,
|
||||
initial_state=initial_state,
|
||||
output_final_state=output_final_state,
|
||||
cu_seqlens=cu_seqlens,
|
||||
chunk_indices=chunk_indices,
|
||||
chunk_size=chunk_size,
|
||||
)
|
||||
|
||||
|
||||
def chunk_kda_with_fused_gate_fwd(
|
||||
q: torch.Tensor,
|
||||
k: torch.Tensor,
|
||||
v: torch.Tensor,
|
||||
raw_g: torch.Tensor,
|
||||
raw_beta: torch.Tensor,
|
||||
A_log: torch.Tensor,
|
||||
g_bias: torch.Tensor | None,
|
||||
scale: float,
|
||||
initial_state: torch.Tensor,
|
||||
output_final_state: bool,
|
||||
lower_bound: float | None = None,
|
||||
cu_seqlens: torch.Tensor | None = None,
|
||||
):
|
||||
chunk_size = FLA_CHUNK_SIZE
|
||||
chunk_indices = (
|
||||
prepare_chunk_indices(cu_seqlens, chunk_size)
|
||||
if cu_seqlens is not None
|
||||
else None
|
||||
)
|
||||
g, beta = fused_kda_gate_chunk_cumsum(
|
||||
raw_g,
|
||||
raw_beta=raw_beta,
|
||||
A_log=A_log,
|
||||
g_bias=g_bias,
|
||||
cu_seqlens=cu_seqlens,
|
||||
chunk_indices=chunk_indices,
|
||||
chunk_size=chunk_size,
|
||||
lower_bound=lower_bound,
|
||||
)
|
||||
return _chunk_kda_fwd_with_cumulative_g(
|
||||
q=q,
|
||||
k=k,
|
||||
v=v,
|
||||
g=g,
|
||||
beta=beta,
|
||||
scale=scale,
|
||||
initial_state=initial_state,
|
||||
output_final_state=output_final_state,
|
||||
cu_seqlens=cu_seqlens,
|
||||
chunk_indices=chunk_indices,
|
||||
chunk_size=chunk_size,
|
||||
safe_gate=lower_bound is not None,
|
||||
)
|
||||
|
||||
|
||||
def chunk_kda(
|
||||
q: torch.Tensor,
|
||||
k: torch.Tensor,
|
||||
v: torch.Tensor,
|
||||
g: torch.Tensor,
|
||||
beta: torch.Tensor,
|
||||
scale: float = None,
|
||||
initial_state: torch.Tensor = None,
|
||||
output_final_state: bool = False,
|
||||
use_qk_l2norm_in_kernel: bool = False,
|
||||
cu_seqlens: torch.Tensor | None = None,
|
||||
**kwargs,
|
||||
):
|
||||
if scale is None:
|
||||
scale = k.shape[-1] ** -0.5
|
||||
|
||||
if use_qk_l2norm_in_kernel:
|
||||
q = l2norm_fwd(q.contiguous())
|
||||
k = l2norm_fwd(k.contiguous())
|
||||
|
||||
o, final_state = chunk_kda_fwd(
|
||||
q=q,
|
||||
k=k,
|
||||
v=v.contiguous(),
|
||||
g=g.contiguous(),
|
||||
beta=beta.contiguous(),
|
||||
scale=scale,
|
||||
initial_state=initial_state.contiguous(),
|
||||
output_final_state=output_final_state,
|
||||
cu_seqlens=cu_seqlens,
|
||||
)
|
||||
return o, final_state
|
||||
|
||||
|
||||
def chunk_kda_with_fused_gate(
|
||||
q: torch.Tensor,
|
||||
k: torch.Tensor,
|
||||
v: torch.Tensor,
|
||||
raw_g: torch.Tensor,
|
||||
raw_beta: torch.Tensor,
|
||||
A_log: torch.Tensor,
|
||||
g_bias: torch.Tensor | None,
|
||||
scale: float | None = None,
|
||||
initial_state: torch.Tensor | None = None,
|
||||
output_final_state: bool = False,
|
||||
lower_bound: float | None = None,
|
||||
use_qk_l2norm_in_kernel: bool = False,
|
||||
cu_seqlens: torch.Tensor | None = None,
|
||||
**kwargs,
|
||||
):
|
||||
"""Run chunk KDA from raw gate and beta projections."""
|
||||
if scale is None:
|
||||
scale = k.shape[-1] ** -0.5
|
||||
|
||||
if use_qk_l2norm_in_kernel:
|
||||
q = l2norm_fwd(q.contiguous())
|
||||
k = l2norm_fwd(k.contiguous())
|
||||
|
||||
o, final_state = chunk_kda_with_fused_gate_fwd(
|
||||
q=q,
|
||||
k=k,
|
||||
v=v.contiguous(),
|
||||
raw_g=raw_g.contiguous(),
|
||||
raw_beta=raw_beta,
|
||||
A_log=A_log,
|
||||
g_bias=g_bias,
|
||||
scale=scale,
|
||||
initial_state=initial_state.contiguous() if initial_state is not None else None,
|
||||
output_final_state=output_final_state,
|
||||
lower_bound=lower_bound,
|
||||
cu_seqlens=cu_seqlens,
|
||||
)
|
||||
return o, final_state
|
||||
|
||||
|
||||
@triton.autotune(
|
||||
configs=[
|
||||
triton.Config({"BT": bt}, num_warps=nw, num_stages=ns)
|
||||
for bt in BT_LIST_AUTOTUNE
|
||||
for nw in NUM_WARPS_AUTOTUNE
|
||||
for ns in [2, 3]
|
||||
],
|
||||
key=["H", "D"],
|
||||
)
|
||||
@triton.jit
|
||||
def kda_gate_fwd_kernel(
|
||||
g,
|
||||
A,
|
||||
y,
|
||||
g_bias,
|
||||
lower_bound,
|
||||
beta: tl.constexpr,
|
||||
threshold: tl.constexpr,
|
||||
T,
|
||||
H,
|
||||
D: tl.constexpr,
|
||||
BT: tl.constexpr,
|
||||
BD: tl.constexpr,
|
||||
HAS_BIAS: tl.constexpr,
|
||||
USE_LOWER_BOUND: tl.constexpr,
|
||||
):
|
||||
i_t, i_h = tl.program_id(0), tl.program_id(1)
|
||||
n_t = i_t * BT
|
||||
|
||||
b_a = tl.exp(tl.load(A + i_h).to(tl.float32))
|
||||
|
||||
stride_row = H * D
|
||||
stride_col = 1
|
||||
|
||||
g_ptr = tl.make_block_ptr(
|
||||
base=g + i_h * D,
|
||||
shape=(T, D),
|
||||
strides=(stride_row, stride_col),
|
||||
offsets=(n_t, 0),
|
||||
block_shape=(BT, BD),
|
||||
order=(1, 0),
|
||||
)
|
||||
|
||||
y_ptr = tl.make_block_ptr(
|
||||
base=y + i_h * D,
|
||||
shape=(T, D),
|
||||
strides=(stride_row, stride_col),
|
||||
offsets=(n_t, 0),
|
||||
block_shape=(BT, BD),
|
||||
order=(1, 0),
|
||||
)
|
||||
|
||||
b_g = tl.load(g_ptr, boundary_check=(0, 1)).to(tl.float32)
|
||||
|
||||
if HAS_BIAS:
|
||||
n_d = tl.arange(0, BD)
|
||||
bias_mask = n_d < D
|
||||
b_bias = tl.load(g_bias + i_h * D + n_d, mask=bias_mask, other=0.0).to(
|
||||
tl.float32
|
||||
)
|
||||
b_g = b_g + b_bias[None, :]
|
||||
|
||||
if USE_LOWER_BOUND:
|
||||
b_y = lower_bound * tl.sigmoid(b_a * b_g)
|
||||
else:
|
||||
g_scaled = b_g * beta
|
||||
use_linear = g_scaled > threshold
|
||||
sp = tl.where(use_linear, b_g, (1.0 / beta) * log(1.0 + tl.exp(g_scaled)))
|
||||
b_y = -b_a * sp
|
||||
|
||||
tl.store(y_ptr, b_y.to(y.dtype.element_ty), boundary_check=(0, 1))
|
||||
|
||||
|
||||
def fused_kda_gate(
|
||||
g: torch.Tensor,
|
||||
A: torch.Tensor,
|
||||
head_k_dim: int,
|
||||
g_bias: torch.Tensor | None = None,
|
||||
beta: float = 1.0,
|
||||
threshold: float = 20.0,
|
||||
lower_bound: float | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Forward pass for KDA gate:
|
||||
input g: [..., H*D]
|
||||
param A: [H] or [1, 1, H, 1]
|
||||
beta: softplus beta parameter
|
||||
threshold: softplus threshold parameter
|
||||
return : [..., H, D]
|
||||
"""
|
||||
orig_shape = g.shape[:-1]
|
||||
|
||||
g = g.view(-1, g.shape[-1])
|
||||
T = g.shape[0]
|
||||
HD = g.shape[1]
|
||||
H = A.numel()
|
||||
assert H * head_k_dim == HD
|
||||
|
||||
y = torch.empty_like(g, dtype=torch.float32)
|
||||
|
||||
def grid(meta):
|
||||
return (cdiv(T, meta["BT"]), H)
|
||||
|
||||
kda_gate_fwd_kernel[grid](
|
||||
g,
|
||||
A,
|
||||
y,
|
||||
g_bias,
|
||||
lower_bound or 0.0,
|
||||
beta,
|
||||
threshold,
|
||||
T,
|
||||
H,
|
||||
head_k_dim,
|
||||
BD=next_power_of_2(head_k_dim),
|
||||
HAS_BIAS=g_bias is not None,
|
||||
USE_LOWER_BOUND=lower_bound is not None,
|
||||
)
|
||||
|
||||
y = y.view(*orig_shape, H, head_k_dim)
|
||||
return y
|
||||
@@ -0,0 +1,662 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# SPDX-FileCopyrightText: Songlin Yang, Yu Zhang, Zhiyuan Li
|
||||
#
|
||||
# This file contains code copied from the flash-linear-attention project.
|
||||
# The original source was licensed under the MIT license.
|
||||
# Copyright (c) 2023-2026, Songlin Yang, Yu Zhang, Zhiyuan Li
|
||||
# Forward-only adaptation of flash-linear-attention 0.5.0.
|
||||
# ruff: noqa: E501
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.third_party.flash_linear_attention.ops.index import prepare_chunk_indices
|
||||
from vllm.third_party.flash_linear_attention.ops.op import exp2, gather
|
||||
from vllm.third_party.flash_linear_attention.ops.utils import is_gather_supported
|
||||
from vllm.triton_utils import tl, triton
|
||||
|
||||
from .chunk_intra_token_parallel import chunk_kda_fwd_intra_token_parallel
|
||||
|
||||
################################################################################
|
||||
# Fused inter + solve_tril kernel: compute off-diagonal Akk and solve in one pass
|
||||
################################################################################
|
||||
|
||||
|
||||
@triton.heuristics(
|
||||
{
|
||||
"IS_VARLEN": lambda args: args["cu_seqlens"] is not None,
|
||||
}
|
||||
)
|
||||
@triton.autotune(
|
||||
configs=[
|
||||
triton.Config({"BK": BK}, num_warps=num_warps)
|
||||
for BK in [32, 64]
|
||||
for num_warps in [1, 2, 4]
|
||||
],
|
||||
key=["H", "HV", "K", "BC"],
|
||||
)
|
||||
@triton.jit(do_not_specialize=["T"])
|
||||
def chunk_kda_fwd_kernel_inter_solve_fused(
|
||||
q,
|
||||
k,
|
||||
g,
|
||||
beta,
|
||||
Aqk,
|
||||
Akkd,
|
||||
Akk,
|
||||
scale,
|
||||
cu_seqlens,
|
||||
chunk_indices,
|
||||
T,
|
||||
H: tl.constexpr,
|
||||
HV: tl.constexpr,
|
||||
K: tl.constexpr,
|
||||
BT: tl.constexpr,
|
||||
BC: tl.constexpr,
|
||||
BK: tl.constexpr,
|
||||
IS_VARLEN: tl.constexpr,
|
||||
USE_SAFE_GATE: tl.constexpr,
|
||||
SOLVE_TRIL_DOT_PRECISION: tl.constexpr,
|
||||
):
|
||||
"""
|
||||
Fused kernel: compute inter-subchunk Akk + solve_tril in one pass.
|
||||
Prerequisite: token_parallel has already computed diagonal Akk blocks in Akkd.
|
||||
|
||||
This kernel:
|
||||
1. Computes off-diagonal Aqk blocks -> writes to global
|
||||
2. Computes off-diagonal Akk blocks -> keeps in registers
|
||||
3. Loads diagonal Akk blocks from Akkd (fp32)
|
||||
4. Does forward substitution on diagonals
|
||||
5. Computes merged Akk_inv
|
||||
6. Writes Akk_inv to Akk
|
||||
"""
|
||||
i_t, i_bh = tl.program_id(0), tl.program_id(1)
|
||||
i_b, i_hv = i_bh // HV, i_bh % HV
|
||||
i_h = i_hv // (HV // H)
|
||||
|
||||
if IS_VARLEN:
|
||||
i_n, i_t = (
|
||||
tl.load(chunk_indices + i_t * 2).to(tl.int32),
|
||||
tl.load(chunk_indices + i_t * 2 + 1).to(tl.int32),
|
||||
)
|
||||
bos, eos = (
|
||||
tl.load(cu_seqlens + i_n).to(tl.int32),
|
||||
tl.load(cu_seqlens + i_n + 1).to(tl.int32),
|
||||
)
|
||||
T = eos - bos
|
||||
else:
|
||||
bos, eos = i_b * T, i_b * T + T
|
||||
|
||||
if i_t * BT >= T:
|
||||
return
|
||||
|
||||
i_tc0 = i_t * BT
|
||||
i_tc1 = i_t * BT + BC
|
||||
i_tc2 = i_t * BT + 2 * BC
|
||||
i_tc3 = i_t * BT + 3 * BC
|
||||
|
||||
q += (bos * H + i_h) * K
|
||||
k += (bos * H + i_h) * K
|
||||
g += (bos * HV + i_hv) * K
|
||||
Aqk += (bos * HV + i_hv) * BT
|
||||
Akk += (bos * HV + i_hv) * BT
|
||||
Akkd += (bos * HV + i_hv) * BC
|
||||
|
||||
o_i = tl.arange(0, BC)
|
||||
m_tc1 = (i_tc1 + o_i) < T
|
||||
m_tc2 = (i_tc2 + o_i) < T
|
||||
m_tc3 = (i_tc3 + o_i) < T
|
||||
|
||||
b_Aqk10 = tl.zeros([BC, BC], dtype=tl.float32)
|
||||
b_Akk10 = tl.zeros([BC, BC], dtype=tl.float32)
|
||||
|
||||
b_Aqk20 = tl.zeros([BC, BC], dtype=tl.float32)
|
||||
b_Akk20 = tl.zeros([BC, BC], dtype=tl.float32)
|
||||
b_Aqk21 = tl.zeros([BC, BC], dtype=tl.float32)
|
||||
b_Akk21 = tl.zeros([BC, BC], dtype=tl.float32)
|
||||
|
||||
b_Aqk30 = tl.zeros([BC, BC], dtype=tl.float32)
|
||||
b_Akk30 = tl.zeros([BC, BC], dtype=tl.float32)
|
||||
b_Aqk31 = tl.zeros([BC, BC], dtype=tl.float32)
|
||||
b_Akk31 = tl.zeros([BC, BC], dtype=tl.float32)
|
||||
b_Aqk32 = tl.zeros([BC, BC], dtype=tl.float32)
|
||||
b_Akk32 = tl.zeros([BC, BC], dtype=tl.float32)
|
||||
|
||||
################################################################################
|
||||
# off-diagonal blocks
|
||||
################################################################################
|
||||
for i_k in range(tl.cdiv(K, BK)):
|
||||
o_k = i_k * BK + tl.arange(0, BK)
|
||||
m_k = o_k < K
|
||||
|
||||
p_k0 = tl.make_block_ptr(
|
||||
k, (T, K), (H * K, 1), (i_tc0, i_k * BK), (BC, BK), (1, 0)
|
||||
)
|
||||
p_g0 = tl.make_block_ptr(
|
||||
g, (T, K), (HV * K, 1), (i_tc0, i_k * BK), (BC, BK), (1, 0)
|
||||
)
|
||||
b_k0 = tl.load(p_k0, boundary_check=(0, 1)).to(tl.float32)
|
||||
b_g0 = tl.load(p_g0, boundary_check=(0, 1)).to(tl.float32)
|
||||
|
||||
if i_tc1 < T:
|
||||
p_q1 = tl.make_block_ptr(
|
||||
q, (T, K), (H * K, 1), (i_tc1, i_k * BK), (BC, BK), (1, 0)
|
||||
)
|
||||
p_k1 = tl.make_block_ptr(
|
||||
k, (T, K), (H * K, 1), (i_tc1, i_k * BK), (BC, BK), (1, 0)
|
||||
)
|
||||
p_g1 = tl.make_block_ptr(
|
||||
g, (T, K), (HV * K, 1), (i_tc1, i_k * BK), (BC, BK), (1, 0)
|
||||
)
|
||||
# [BC, BK]
|
||||
b_q1 = tl.load(p_q1, boundary_check=(0, 1)).to(tl.float32)
|
||||
b_k1 = tl.load(p_k1, boundary_check=(0, 1)).to(tl.float32)
|
||||
b_g1 = tl.load(p_g1, boundary_check=(0, 1)).to(tl.float32)
|
||||
# [BK]
|
||||
b_gn1 = tl.load(g + i_tc1 * HV * K + o_k, mask=m_k, other=0).to(tl.float32)
|
||||
# [BC, BK]
|
||||
b_gqn = tl.where(m_tc1[:, None], exp2(b_g1 - b_gn1[None, :]), 0)
|
||||
# [BK, BC]
|
||||
b_kgt = tl.trans(b_k0 * exp2(b_gn1[None, :] - b_g0))
|
||||
# [BC, BC]
|
||||
b_Aqk10 += tl.dot(b_q1 * b_gqn, b_kgt)
|
||||
b_Akk10 += tl.dot(b_k1 * b_gqn, b_kgt)
|
||||
|
||||
if i_tc2 < T:
|
||||
p_q2 = tl.make_block_ptr(
|
||||
q, (T, K), (H * K, 1), (i_tc2, i_k * BK), (BC, BK), (1, 0)
|
||||
)
|
||||
p_k2 = tl.make_block_ptr(
|
||||
k, (T, K), (H * K, 1), (i_tc2, i_k * BK), (BC, BK), (1, 0)
|
||||
)
|
||||
p_g2 = tl.make_block_ptr(
|
||||
g, (T, K), (HV * K, 1), (i_tc2, i_k * BK), (BC, BK), (1, 0)
|
||||
)
|
||||
# [BC, BK]
|
||||
b_q2 = tl.load(p_q2, boundary_check=(0, 1)).to(tl.float32)
|
||||
b_k2 = tl.load(p_k2, boundary_check=(0, 1)).to(tl.float32)
|
||||
b_g2 = tl.load(p_g2, boundary_check=(0, 1)).to(tl.float32)
|
||||
# [BK]
|
||||
b_gn2 = tl.load(g + i_tc2 * HV * K + o_k, mask=m_k, other=0).to(
|
||||
tl.float32
|
||||
)
|
||||
# [BC, BK]
|
||||
b_gqn2 = tl.where(m_tc2[:, None], exp2(b_g2 - b_gn2[None, :]), 0)
|
||||
b_qg2 = b_q2 * b_gqn2
|
||||
b_kg2 = b_k2 * b_gqn2
|
||||
# [BK, BC]
|
||||
b_kgt = tl.trans(b_k0 * exp2(b_gn2[None, :] - b_g0))
|
||||
b_Aqk20 += tl.dot(b_qg2, b_kgt)
|
||||
b_Akk20 += tl.dot(b_kg2, b_kgt)
|
||||
# [BC, BC]
|
||||
b_kgt = tl.trans(b_k1 * exp2(b_gn2[None, :] - b_g1))
|
||||
# [BC, BC]
|
||||
b_Aqk21 += tl.dot(b_qg2, b_kgt)
|
||||
b_Akk21 += tl.dot(b_kg2, b_kgt)
|
||||
|
||||
if i_tc3 < T:
|
||||
p_q3 = tl.make_block_ptr(
|
||||
q, (T, K), (H * K, 1), (i_tc3, i_k * BK), (BC, BK), (1, 0)
|
||||
)
|
||||
p_k3 = tl.make_block_ptr(
|
||||
k, (T, K), (H * K, 1), (i_tc3, i_k * BK), (BC, BK), (1, 0)
|
||||
)
|
||||
p_g3 = tl.make_block_ptr(
|
||||
g, (T, K), (HV * K, 1), (i_tc3, i_k * BK), (BC, BK), (1, 0)
|
||||
)
|
||||
# [BC, BK]
|
||||
b_q3 = tl.load(p_q3, boundary_check=(0, 1)).to(tl.float32)
|
||||
b_k3 = tl.load(p_k3, boundary_check=(0, 1)).to(tl.float32)
|
||||
b_g3 = tl.load(p_g3, boundary_check=(0, 1)).to(tl.float32)
|
||||
# [BK]
|
||||
b_gn3 = tl.load(g + i_tc3 * HV * K + o_k, mask=m_k, other=0).to(
|
||||
tl.float32
|
||||
)
|
||||
# [BC, BK]
|
||||
b_gqn3 = tl.where(m_tc3[:, None], exp2(b_g3 - b_gn3[None, :]), 0)
|
||||
b_qg3 = b_q3 * b_gqn3
|
||||
b_kg3 = b_k3 * b_gqn3
|
||||
# [BK, BC]
|
||||
b_kgt = tl.trans(b_k0 * exp2(b_gn3[None, :] - b_g0))
|
||||
# [BC, BC]
|
||||
b_Aqk30 += tl.dot(b_qg3, b_kgt)
|
||||
b_Akk30 += tl.dot(b_kg3, b_kgt)
|
||||
# [BK, BC]
|
||||
b_kgt = tl.trans(b_k1 * exp2(b_gn3[None, :] - b_g1))
|
||||
# [BC, BC]
|
||||
b_Aqk31 += tl.dot(b_qg3, b_kgt)
|
||||
b_Akk31 += tl.dot(b_kg3, b_kgt)
|
||||
# [BK, BC]
|
||||
b_kgt = tl.trans(b_k2 * exp2(b_gn3[None, :] - b_g2))
|
||||
# [BC, BC]
|
||||
b_Aqk32 += tl.dot(b_qg3, b_kgt)
|
||||
b_Akk32 += tl.dot(b_kg3, b_kgt)
|
||||
|
||||
################################################################################
|
||||
# save off-diagonal Aqk blocks and prepare Akk
|
||||
################################################################################
|
||||
if i_tc1 < T:
|
||||
p_Aqk10 = tl.make_block_ptr(
|
||||
Aqk, (T, BT), (HV * BT, 1), (i_tc1, 0), (BC, BC), (1, 0)
|
||||
)
|
||||
tl.store(
|
||||
p_Aqk10, (b_Aqk10 * scale).to(Aqk.dtype.element_ty), boundary_check=(0, 1)
|
||||
)
|
||||
|
||||
p_b1 = tl.make_block_ptr(
|
||||
beta + bos * HV + i_hv, (T,), (HV,), (i_tc1,), (BC,), (0,)
|
||||
)
|
||||
b_b1 = tl.load(p_b1, boundary_check=(0,)).to(tl.float32)
|
||||
b_Akk10 = b_Akk10 * b_b1[:, None]
|
||||
if i_tc2 < T:
|
||||
p_Aqk20 = tl.make_block_ptr(
|
||||
Aqk, (T, BT), (HV * BT, 1), (i_tc2, 0), (BC, BC), (1, 0)
|
||||
)
|
||||
p_Aqk21 = tl.make_block_ptr(
|
||||
Aqk, (T, BT), (HV * BT, 1), (i_tc2, BC), (BC, BC), (1, 0)
|
||||
)
|
||||
tl.store(
|
||||
p_Aqk20, (b_Aqk20 * scale).to(Aqk.dtype.element_ty), boundary_check=(0, 1)
|
||||
)
|
||||
tl.store(
|
||||
p_Aqk21, (b_Aqk21 * scale).to(Aqk.dtype.element_ty), boundary_check=(0, 1)
|
||||
)
|
||||
|
||||
p_b2 = tl.make_block_ptr(
|
||||
beta + bos * HV + i_hv, (T,), (HV,), (i_tc2,), (BC,), (0,)
|
||||
)
|
||||
b_b2 = tl.load(p_b2, boundary_check=(0,)).to(tl.float32)
|
||||
b_Akk20 = b_Akk20 * b_b2[:, None]
|
||||
b_Akk21 = b_Akk21 * b_b2[:, None]
|
||||
if i_tc3 < T:
|
||||
p_Aqk30 = tl.make_block_ptr(
|
||||
Aqk, (T, BT), (HV * BT, 1), (i_tc3, 0), (BC, BC), (1, 0)
|
||||
)
|
||||
p_Aqk31 = tl.make_block_ptr(
|
||||
Aqk, (T, BT), (HV * BT, 1), (i_tc3, BC), (BC, BC), (1, 0)
|
||||
)
|
||||
p_Aqk32 = tl.make_block_ptr(
|
||||
Aqk, (T, BT), (HV * BT, 1), (i_tc3, 2 * BC), (BC, BC), (1, 0)
|
||||
)
|
||||
tl.store(
|
||||
p_Aqk30, (b_Aqk30 * scale).to(Aqk.dtype.element_ty), boundary_check=(0, 1)
|
||||
)
|
||||
tl.store(
|
||||
p_Aqk31, (b_Aqk31 * scale).to(Aqk.dtype.element_ty), boundary_check=(0, 1)
|
||||
)
|
||||
tl.store(
|
||||
p_Aqk32, (b_Aqk32 * scale).to(Aqk.dtype.element_ty), boundary_check=(0, 1)
|
||||
)
|
||||
|
||||
p_b3 = tl.make_block_ptr(
|
||||
beta + bos * HV + i_hv, (T,), (HV,), (i_tc3,), (BC,), (0,)
|
||||
)
|
||||
b_b3 = tl.load(p_b3, boundary_check=(0,)).to(tl.float32)
|
||||
b_Akk30 = b_Akk30 * b_b3[:, None]
|
||||
b_Akk31 = b_Akk31 * b_b3[:, None]
|
||||
b_Akk32 = b_Akk32 * b_b3[:, None]
|
||||
|
||||
p_Akk00 = tl.make_block_ptr(
|
||||
Akkd, (T, BC), (HV * BC, 1), (i_tc0, 0), (BC, BC), (1, 0)
|
||||
)
|
||||
p_Akk11 = tl.make_block_ptr(
|
||||
Akkd, (T, BC), (HV * BC, 1), (i_tc1, 0), (BC, BC), (1, 0)
|
||||
)
|
||||
p_Akk22 = tl.make_block_ptr(
|
||||
Akkd, (T, BC), (HV * BC, 1), (i_tc2, 0), (BC, BC), (1, 0)
|
||||
)
|
||||
p_Akk33 = tl.make_block_ptr(
|
||||
Akkd, (T, BC), (HV * BC, 1), (i_tc3, 0), (BC, BC), (1, 0)
|
||||
)
|
||||
b_Ai00 = tl.load(p_Akk00, boundary_check=(0, 1)).to(tl.float32)
|
||||
b_Ai11 = tl.load(p_Akk11, boundary_check=(0, 1)).to(tl.float32)
|
||||
b_Ai22 = tl.load(p_Akk22, boundary_check=(0, 1)).to(tl.float32)
|
||||
b_Ai33 = tl.load(p_Akk33, boundary_check=(0, 1)).to(tl.float32)
|
||||
|
||||
################################################################################
|
||||
# forward substitution on diagonals
|
||||
################################################################################
|
||||
|
||||
if not USE_SAFE_GATE:
|
||||
m_A = o_i[:, None] > o_i[None, :]
|
||||
m_I = o_i[:, None] == o_i[None, :]
|
||||
|
||||
b_Ai00 = -tl.where(m_A, b_Ai00, 0)
|
||||
b_Ai11 = -tl.where(m_A, b_Ai11, 0)
|
||||
b_Ai22 = -tl.where(m_A, b_Ai22, 0)
|
||||
b_Ai33 = -tl.where(m_A, b_Ai33, 0)
|
||||
|
||||
for i in range(2, min(BC, T - i_tc0)):
|
||||
b_a00 = -tl.load(Akkd + (i_tc0 + i) * HV * BC + o_i)
|
||||
b_a00 = tl.where(o_i < i, b_a00, 0.0)
|
||||
b_a00 += tl.sum(b_a00[:, None] * b_Ai00, 0)
|
||||
b_Ai00 = tl.where((o_i == i)[:, None], b_a00, b_Ai00)
|
||||
for i in range(BC + 2, min(2 * BC, T - i_tc0)):
|
||||
b_a11 = -tl.load(Akkd + (i_tc0 + i) * HV * BC + o_i)
|
||||
b_a11 = tl.where(o_i < i - BC, b_a11, 0.0)
|
||||
b_a11 += tl.sum(b_a11[:, None] * b_Ai11, 0)
|
||||
b_Ai11 = tl.where((o_i == i - BC)[:, None], b_a11, b_Ai11)
|
||||
for i in range(2 * BC + 2, min(3 * BC, T - i_tc0)):
|
||||
b_a22 = -tl.load(Akkd + (i_tc0 + i) * HV * BC + o_i)
|
||||
b_a22 = tl.where(o_i < i - 2 * BC, b_a22, 0.0)
|
||||
b_a22 += tl.sum(b_a22[:, None] * b_Ai22, 0)
|
||||
b_Ai22 = tl.where((o_i == i - 2 * BC)[:, None], b_a22, b_Ai22)
|
||||
for i in range(3 * BC + 2, min(4 * BC, T - i_tc0)):
|
||||
b_a33 = -tl.load(Akkd + (i_tc0 + i) * HV * BC + o_i)
|
||||
b_a33 = tl.where(o_i < i - 3 * BC, b_a33, 0.0)
|
||||
b_a33 += tl.sum(b_a33[:, None] * b_Ai33, 0)
|
||||
b_Ai33 = tl.where((o_i == i - 3 * BC)[:, None], b_a33, b_Ai33)
|
||||
|
||||
b_Ai00 += m_I
|
||||
b_Ai11 += m_I
|
||||
b_Ai22 += m_I
|
||||
b_Ai33 += m_I
|
||||
|
||||
################################################################################
|
||||
# compute merged inverse using off-diagonals
|
||||
################################################################################
|
||||
|
||||
# we used tf32 to maintain matrix inverse's precision whenever possible.
|
||||
b_Ai10 = -tl.dot(
|
||||
tl.dot(b_Ai11, b_Akk10, input_precision=SOLVE_TRIL_DOT_PRECISION),
|
||||
b_Ai00,
|
||||
input_precision=SOLVE_TRIL_DOT_PRECISION,
|
||||
)
|
||||
b_Ai21 = -tl.dot(
|
||||
tl.dot(b_Ai22, b_Akk21, input_precision=SOLVE_TRIL_DOT_PRECISION),
|
||||
b_Ai11,
|
||||
input_precision=SOLVE_TRIL_DOT_PRECISION,
|
||||
)
|
||||
b_Ai32 = -tl.dot(
|
||||
tl.dot(b_Ai33, b_Akk32, input_precision=SOLVE_TRIL_DOT_PRECISION),
|
||||
b_Ai22,
|
||||
input_precision=SOLVE_TRIL_DOT_PRECISION,
|
||||
)
|
||||
|
||||
b_Ai20 = -tl.dot(
|
||||
b_Ai22,
|
||||
tl.dot(b_Akk20, b_Ai00, input_precision=SOLVE_TRIL_DOT_PRECISION)
|
||||
+ tl.dot(b_Akk21, b_Ai10, input_precision=SOLVE_TRIL_DOT_PRECISION),
|
||||
input_precision=SOLVE_TRIL_DOT_PRECISION,
|
||||
)
|
||||
b_Ai31 = -tl.dot(
|
||||
b_Ai33,
|
||||
tl.dot(b_Akk31, b_Ai11, input_precision=SOLVE_TRIL_DOT_PRECISION)
|
||||
+ tl.dot(b_Akk32, b_Ai21, input_precision=SOLVE_TRIL_DOT_PRECISION),
|
||||
input_precision=SOLVE_TRIL_DOT_PRECISION,
|
||||
)
|
||||
b_Ai30 = -tl.dot(
|
||||
b_Ai33,
|
||||
tl.dot(b_Akk30, b_Ai00, input_precision=SOLVE_TRIL_DOT_PRECISION)
|
||||
+ tl.dot(b_Akk31, b_Ai10, input_precision=SOLVE_TRIL_DOT_PRECISION)
|
||||
+ tl.dot(b_Akk32, b_Ai20, input_precision=SOLVE_TRIL_DOT_PRECISION),
|
||||
input_precision=SOLVE_TRIL_DOT_PRECISION,
|
||||
)
|
||||
|
||||
################################################################################
|
||||
# store full Akk_inv to Akk
|
||||
################################################################################
|
||||
|
||||
p_Akk00 = tl.make_block_ptr(
|
||||
Akk, (T, BT), (HV * BT, 1), (i_tc0, 0), (BC, BC), (1, 0)
|
||||
)
|
||||
p_Akk10 = tl.make_block_ptr(
|
||||
Akk, (T, BT), (HV * BT, 1), (i_tc1, 0), (BC, BC), (1, 0)
|
||||
)
|
||||
p_Akk11 = tl.make_block_ptr(
|
||||
Akk, (T, BT), (HV * BT, 1), (i_tc1, BC), (BC, BC), (1, 0)
|
||||
)
|
||||
p_Akk20 = tl.make_block_ptr(
|
||||
Akk, (T, BT), (HV * BT, 1), (i_tc2, 0), (BC, BC), (1, 0)
|
||||
)
|
||||
p_Akk21 = tl.make_block_ptr(
|
||||
Akk, (T, BT), (HV * BT, 1), (i_tc2, BC), (BC, BC), (1, 0)
|
||||
)
|
||||
p_Akk22 = tl.make_block_ptr(
|
||||
Akk, (T, BT), (HV * BT, 1), (i_tc2, 2 * BC), (BC, BC), (1, 0)
|
||||
)
|
||||
p_Akk30 = tl.make_block_ptr(
|
||||
Akk, (T, BT), (HV * BT, 1), (i_tc3, 0), (BC, BC), (1, 0)
|
||||
)
|
||||
p_Akk31 = tl.make_block_ptr(
|
||||
Akk, (T, BT), (HV * BT, 1), (i_tc3, BC), (BC, BC), (1, 0)
|
||||
)
|
||||
p_Akk32 = tl.make_block_ptr(
|
||||
Akk, (T, BT), (HV * BT, 1), (i_tc3, 2 * BC), (BC, BC), (1, 0)
|
||||
)
|
||||
p_Akk33 = tl.make_block_ptr(
|
||||
Akk, (T, BT), (HV * BT, 1), (i_tc3, 3 * BC), (BC, BC), (1, 0)
|
||||
)
|
||||
|
||||
tl.store(p_Akk00, b_Ai00.to(Akk.dtype.element_ty), boundary_check=(0, 1))
|
||||
tl.store(p_Akk10, b_Ai10.to(Akk.dtype.element_ty), boundary_check=(0, 1))
|
||||
tl.store(p_Akk11, b_Ai11.to(Akk.dtype.element_ty), boundary_check=(0, 1))
|
||||
tl.store(p_Akk20, b_Ai20.to(Akk.dtype.element_ty), boundary_check=(0, 1))
|
||||
tl.store(p_Akk21, b_Ai21.to(Akk.dtype.element_ty), boundary_check=(0, 1))
|
||||
tl.store(p_Akk22, b_Ai22.to(Akk.dtype.element_ty), boundary_check=(0, 1))
|
||||
tl.store(p_Akk30, b_Ai30.to(Akk.dtype.element_ty), boundary_check=(0, 1))
|
||||
tl.store(p_Akk31, b_Ai31.to(Akk.dtype.element_ty), boundary_check=(0, 1))
|
||||
tl.store(p_Akk32, b_Ai32.to(Akk.dtype.element_ty), boundary_check=(0, 1))
|
||||
tl.store(p_Akk33, b_Ai33.to(Akk.dtype.element_ty), boundary_check=(0, 1))
|
||||
|
||||
|
||||
@triton.heuristics(
|
||||
{
|
||||
"IS_VARLEN": lambda args: args["cu_seqlens"] is not None,
|
||||
}
|
||||
)
|
||||
@triton.autotune(
|
||||
configs=[
|
||||
triton.Config({}, num_warps=num_warps, num_stages=num_stages)
|
||||
for num_warps in [1, 2, 4, 8]
|
||||
for num_stages in [2, 3, 4]
|
||||
],
|
||||
key=["BK", "NC", "BT", "HV"],
|
||||
)
|
||||
@triton.jit(do_not_specialize=["B", "T"])
|
||||
def chunk_kda_fwd_kernel_intra_sub_chunk(
|
||||
q,
|
||||
k,
|
||||
g,
|
||||
beta,
|
||||
Aqk,
|
||||
Akk,
|
||||
scale,
|
||||
cu_seqlens,
|
||||
chunk_indices,
|
||||
T,
|
||||
H: tl.constexpr,
|
||||
HV: tl.constexpr,
|
||||
K: tl.constexpr,
|
||||
BT: tl.constexpr,
|
||||
BC: tl.constexpr,
|
||||
BK: tl.constexpr,
|
||||
IS_VARLEN: tl.constexpr,
|
||||
USE_GATHER: tl.constexpr,
|
||||
):
|
||||
i_t, i_i, i_bh = tl.program_id(0), tl.program_id(1), tl.program_id(2)
|
||||
i_b, i_hv = i_bh // HV, i_bh % HV
|
||||
i_h = i_hv // (HV // H)
|
||||
|
||||
if IS_VARLEN:
|
||||
i_n, i_t = (
|
||||
tl.load(chunk_indices + i_t * 2).to(tl.int32),
|
||||
tl.load(chunk_indices + i_t * 2 + 1).to(tl.int32),
|
||||
)
|
||||
bos, eos = (
|
||||
tl.load(cu_seqlens + i_n).to(tl.int32),
|
||||
tl.load(cu_seqlens + i_n + 1).to(tl.int32),
|
||||
)
|
||||
T = eos - bos
|
||||
else:
|
||||
bos, eos = i_b * T, i_b * T + T
|
||||
|
||||
i_ti = i_t * BT + i_i * BC
|
||||
if i_ti >= T:
|
||||
return
|
||||
|
||||
o_c = i_ti + tl.arange(0, BC)
|
||||
m_c = o_c < T
|
||||
|
||||
q = q + (bos * H + i_h) * K
|
||||
k = k + (bos * H + i_h) * K
|
||||
g = g + (bos * HV + i_hv) * K
|
||||
beta = beta + bos * HV + i_hv
|
||||
Aqk = Aqk + (bos * HV + i_hv) * BT
|
||||
Akk = Akk + (bos * HV + i_hv) * BC
|
||||
|
||||
p_q = tl.make_block_ptr(q, (T, K), (H * K, 1), (i_ti, 0), (BC, BK), (1, 0))
|
||||
p_k = tl.make_block_ptr(k, (T, K), (H * K, 1), (i_ti, 0), (BC, BK), (1, 0))
|
||||
p_g = tl.make_block_ptr(g, (T, K), (HV * K, 1), (i_ti, 0), (BC, BK), (1, 0))
|
||||
|
||||
p_beta = tl.make_block_ptr(beta, (T,), (HV,), (i_ti,), (BC,), (0,))
|
||||
|
||||
b_q = tl.load(p_q, boundary_check=(0, 1))
|
||||
b_k = tl.load(p_k, boundary_check=(0, 1))
|
||||
b_g = tl.load(p_g, boundary_check=(0, 1))
|
||||
b_beta = tl.load(p_beta, boundary_check=(0,)).to(tl.float32)
|
||||
|
||||
if USE_GATHER:
|
||||
b_gn = gather(
|
||||
b_g, tl.full([1, BK], min(BC // 2, T - i_ti - 1), dtype=tl.int16), axis=0
|
||||
)
|
||||
else:
|
||||
# caculate offset
|
||||
p_gn = g + (i_ti + min(BC // 2, T - i_ti - 1)) * HV * K + tl.arange(0, BK)
|
||||
b_gn = tl.load(p_gn, mask=tl.arange(0, BK) < K, other=0.0)
|
||||
b_gn = b_gn[None, :]
|
||||
|
||||
# current block, keep numerical stability by subtracting the left boundary
|
||||
# less than 85 to avoid overflow in exp2
|
||||
b_gm = (b_g - b_gn).to(tl.float32)
|
||||
|
||||
b_gq = tl.where(m_c[:, None], exp2(b_gm), 0.0)
|
||||
b_gk = tl.where(m_c[:, None], exp2(-b_gm), 0.0)
|
||||
|
||||
b_kgt = tl.trans(b_k * b_gk)
|
||||
|
||||
b_Aqk = tl.dot(b_q * b_gq, b_kgt) * scale
|
||||
b_Akk = tl.dot(b_k * b_gq, b_kgt) * b_beta[:, None]
|
||||
|
||||
o_i = tl.arange(0, BC)
|
||||
m_Aqk = o_i[:, None] >= o_i[None, :]
|
||||
m_Akk = o_i[:, None] > o_i[None, :]
|
||||
m_I = o_i[:, None] == o_i[None, :]
|
||||
|
||||
b_Aqk = tl.where(m_Aqk, b_Aqk, 0.0)
|
||||
b_Akk = tl.where(m_Akk, b_Akk, 0.0)
|
||||
|
||||
p_Aqk = tl.make_block_ptr(
|
||||
Aqk, (T, BT), (HV * BT, 1), (i_ti, i_i * BC), (BC, BC), (1, 0)
|
||||
)
|
||||
p_Akk = tl.make_block_ptr(Akk, (T, BC), (HV * BC, 1), (i_ti, 0), (BC, BC), (1, 0))
|
||||
tl.store(p_Aqk, b_Aqk.to(Aqk.dtype.element_ty), boundary_check=(0, 1))
|
||||
tl.store(p_Akk, b_Akk.to(Akk.dtype.element_ty), boundary_check=(0, 1))
|
||||
|
||||
tl.debug_barrier()
|
||||
|
||||
################################################################################
|
||||
# forward substitution
|
||||
################################################################################
|
||||
|
||||
b_Ai = -b_Akk
|
||||
for i in range(2, min(BC, T - i_ti)):
|
||||
b_a = -tl.load(Akk + (i_ti + i) * HV * BC + o_i)
|
||||
b_a = tl.where(o_i < i, b_a, 0.0)
|
||||
b_a += tl.sum(b_a[:, None] * b_Ai, 0)
|
||||
b_Ai = tl.where((o_i == i)[:, None], b_a, b_Ai)
|
||||
b_Ai += m_I
|
||||
tl.store(p_Akk, b_Ai.to(Akk.dtype.element_ty), boundary_check=(0, 1))
|
||||
|
||||
|
||||
def chunk_kda_fwd_intra(
|
||||
q: torch.Tensor,
|
||||
k: torch.Tensor,
|
||||
gk: torch.Tensor | None = None,
|
||||
beta: torch.Tensor | None = None,
|
||||
scale: float | None = None,
|
||||
cu_seqlens: torch.LongTensor | None = None,
|
||||
chunk_size: int = 64,
|
||||
chunk_indices: torch.LongTensor | None = None,
|
||||
safe_gate: bool = False,
|
||||
):
|
||||
B, T, H, K, HV = *k.shape, gk.shape[2]
|
||||
BT = chunk_size
|
||||
BC = 16
|
||||
if chunk_indices is None and cu_seqlens is not None:
|
||||
chunk_indices = prepare_chunk_indices(cu_seqlens, BT)
|
||||
NT = triton.cdiv(T, BT) if cu_seqlens is None else len(chunk_indices)
|
||||
NC = triton.cdiv(BT, BC)
|
||||
|
||||
Aqk = torch.empty(B, T, HV, BT, device=k.device, dtype=k.dtype)
|
||||
# Akk must be zero-initialized - kernel only writes lower triangular
|
||||
Akk = torch.zeros(B, T, HV, BT, device=k.device, dtype=k.dtype)
|
||||
# Separate fp32 buffer for diagonal 16x16 blocks (for precision in solve_tril)
|
||||
Akkd = torch.empty(B, T, HV, BC, device=k.device, dtype=torch.float32)
|
||||
|
||||
# Compute diagonal blocks into Akkd in fp32.
|
||||
if safe_gate:
|
||||
grid = (NT, NC, B * HV)
|
||||
BK = triton.next_power_of_2(K)
|
||||
chunk_kda_fwd_kernel_intra_sub_chunk[grid](
|
||||
q=q,
|
||||
k=k,
|
||||
g=gk,
|
||||
beta=beta,
|
||||
Aqk=Aqk,
|
||||
Akk=Akkd,
|
||||
scale=scale,
|
||||
cu_seqlens=cu_seqlens,
|
||||
chunk_indices=chunk_indices,
|
||||
T=T,
|
||||
H=H,
|
||||
HV=HV,
|
||||
K=K,
|
||||
BT=BT,
|
||||
BC=BC,
|
||||
BK=BK,
|
||||
USE_GATHER=is_gather_supported,
|
||||
)
|
||||
else:
|
||||
Aqk, Akkd = chunk_kda_fwd_intra_token_parallel(
|
||||
q=q,
|
||||
k=k,
|
||||
gk=gk,
|
||||
beta=beta,
|
||||
Aqk=Aqk,
|
||||
Akk=Akkd,
|
||||
scale=scale,
|
||||
cu_seqlens=cu_seqlens,
|
||||
chunk_size=BT,
|
||||
sub_chunk_size=BC,
|
||||
)
|
||||
|
||||
# Step 2: Fused inter + solve_tril (works for both fixed-len and varlen)
|
||||
solve_tril_dot_precision = (
|
||||
"tf32"
|
||||
if current_platform.is_cuda() and current_platform.has_device_capability(80)
|
||||
else "ieee"
|
||||
)
|
||||
grid = (NT, B * HV)
|
||||
chunk_kda_fwd_kernel_inter_solve_fused[grid](
|
||||
q=q,
|
||||
k=k,
|
||||
g=gk,
|
||||
beta=beta,
|
||||
Aqk=Aqk,
|
||||
Akkd=Akkd,
|
||||
Akk=Akk,
|
||||
scale=scale,
|
||||
cu_seqlens=cu_seqlens,
|
||||
chunk_indices=chunk_indices,
|
||||
T=T,
|
||||
H=H,
|
||||
HV=HV,
|
||||
K=K,
|
||||
BT=BT,
|
||||
BC=BC,
|
||||
USE_SAFE_GATE=safe_gate,
|
||||
SOLVE_TRIL_DOT_PRECISION=solve_tril_dot_precision,
|
||||
)
|
||||
return Aqk, Akk
|
||||
@@ -0,0 +1,197 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# SPDX-FileCopyrightText: Songlin Yang, Yu Zhang, Zhiyuan Li
|
||||
#
|
||||
# This file contains code copied from the flash-linear-attention project.
|
||||
# The original source was licensed under the MIT license.
|
||||
# Copyright (c) 2023-2026, Songlin Yang, Yu Zhang, Zhiyuan Li
|
||||
# Forward-only adaptation of flash-linear-attention 0.5.0.
|
||||
# ruff: noqa: E501
|
||||
|
||||
# Token-parallel implementation of KDA intra chunk kernel
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.third_party.flash_linear_attention.ops.op import exp2
|
||||
from vllm.triton_utils import tl, triton
|
||||
|
||||
|
||||
@triton.heuristics(
|
||||
{
|
||||
"IS_VARLEN": lambda args: args["cu_seqlens"] is not None,
|
||||
}
|
||||
)
|
||||
@triton.autotune(
|
||||
configs=[
|
||||
triton.Config({"BH": BH}, num_warps=num_warps)
|
||||
for BH in [1, 2, 4, 8]
|
||||
for num_warps in [1, 2, 4, 8]
|
||||
],
|
||||
key=["K", "H", "HV"],
|
||||
)
|
||||
@triton.jit(do_not_specialize=["T", "N"])
|
||||
def chunk_kda_fwd_kernel_intra_token_parallel(
|
||||
q,
|
||||
k,
|
||||
g,
|
||||
beta,
|
||||
Aqk,
|
||||
Akk,
|
||||
scale,
|
||||
cu_seqlens,
|
||||
N,
|
||||
T,
|
||||
H: tl.constexpr,
|
||||
HV: tl.constexpr,
|
||||
K: tl.constexpr,
|
||||
BT: tl.constexpr,
|
||||
BC: tl.constexpr,
|
||||
BH: tl.constexpr,
|
||||
IS_VARLEN: tl.constexpr,
|
||||
):
|
||||
i_tg, i_hg = tl.program_id(0), tl.program_id(1)
|
||||
|
||||
if IS_VARLEN:
|
||||
i_n = 0
|
||||
left, right = 0, N
|
||||
|
||||
# Unrolled binary search (max B=2^32)
|
||||
# We can limit iterations based on expected max batch size if needed
|
||||
# 20 iterations covers B=1M, usually enough
|
||||
for _ in range(20):
|
||||
if left < right:
|
||||
mid = (left + right) // 2
|
||||
if i_tg < tl.load(cu_seqlens + mid + 1).to(tl.int32):
|
||||
right = mid
|
||||
else:
|
||||
left = mid + 1
|
||||
i_n = left
|
||||
|
||||
bos, eos = (
|
||||
tl.load(cu_seqlens + i_n).to(tl.int32),
|
||||
tl.load(cu_seqlens + i_n + 1).to(tl.int32),
|
||||
)
|
||||
T = eos - bos
|
||||
i_t = i_tg - bos
|
||||
else:
|
||||
bos = (i_tg // T) * T
|
||||
i_t = i_tg % T
|
||||
|
||||
if i_t >= T:
|
||||
return
|
||||
|
||||
i_c = i_t // BT
|
||||
i_s = (i_t % BT) // BC
|
||||
i_tc = i_c * BT
|
||||
i_ts = i_tc + i_s * BC
|
||||
|
||||
G: tl.constexpr = HV // H
|
||||
|
||||
q += bos * H * K
|
||||
k += bos * H * K
|
||||
g += bos * HV * K
|
||||
Aqk += bos * HV * BT
|
||||
Akk += bos * HV * BC
|
||||
beta += bos * HV
|
||||
|
||||
BK: tl.constexpr = triton.next_power_of_2(K)
|
||||
o_hv = i_hg * BH + tl.arange(0, BH)
|
||||
o_h = o_hv // G
|
||||
o_k = tl.arange(0, BK)
|
||||
m_hv = o_hv < HV
|
||||
m_k = o_k < K
|
||||
m_hk = m_hv[:, None] & m_k[None, :]
|
||||
|
||||
# q/k: [B, T, H, K], manual load via mapped qk head index
|
||||
p_qk = o_h[:, None] * K + o_k[None, :]
|
||||
b_q = tl.load(q + i_t * H * K + p_qk, mask=m_hk, other=0).to(tl.float32)
|
||||
b_k = tl.load(k + i_t * H * K + p_qk, mask=m_hk, other=0).to(tl.float32)
|
||||
|
||||
# g: [B, T, HV, K], beta: [B, T, HV]
|
||||
p_g = tl.make_block_ptr(
|
||||
g + i_t * HV * K, (HV, K), (K, 1), (i_hg * BH, 0), (BH, BK), (1, 0)
|
||||
)
|
||||
p_beta = tl.make_block_ptr(beta + i_t * HV, (HV,), (1,), (i_hg * BH,), (BH,), (0,))
|
||||
b_g = tl.load(p_g, boundary_check=(0, 1)).to(tl.float32)
|
||||
b_beta = tl.load(p_beta, boundary_check=(0,)).to(tl.float32)
|
||||
b_k *= b_beta[:, None]
|
||||
|
||||
for j in range(i_ts, min(i_t + 1, min(T, i_ts + BC))):
|
||||
b_kj = tl.load(k + j * H * K + p_qk, mask=m_hk, other=0).to(tl.float32)
|
||||
p_gj = tl.make_block_ptr(
|
||||
g + j * HV * K, (HV, K), (K, 1), (i_hg * BH, 0), (BH, BK), (1, 0)
|
||||
)
|
||||
b_gj = tl.load(p_gj, boundary_check=(0, 1)).to(tl.float32)
|
||||
|
||||
b_kgj = tl.where(m_k[None, :], b_kj * exp2(b_g - b_gj), 0.0)
|
||||
b_Aqk = tl.sum(b_q * b_kgj, axis=1) * scale
|
||||
b_Akk = tl.sum(b_k * b_kgj, axis=1) * tl.where(j < i_t, 1.0, 0.0)
|
||||
|
||||
tl.store(
|
||||
Aqk + i_t * HV * BT + o_hv * BT + j % BT,
|
||||
b_Aqk.to(Aqk.dtype.element_ty),
|
||||
mask=m_hv,
|
||||
)
|
||||
tl.store(
|
||||
Akk + i_t * HV * BC + o_hv * BC + j - i_ts,
|
||||
b_Akk.to(Akk.dtype.element_ty),
|
||||
mask=m_hv,
|
||||
)
|
||||
|
||||
|
||||
def chunk_kda_fwd_intra_token_parallel(
|
||||
q: torch.Tensor,
|
||||
k: torch.Tensor,
|
||||
gk: torch.Tensor,
|
||||
beta: torch.Tensor,
|
||||
Aqk: torch.Tensor,
|
||||
Akk: torch.Tensor,
|
||||
scale: float,
|
||||
cu_seqlens: torch.LongTensor | None = None,
|
||||
chunk_size: int = 64,
|
||||
sub_chunk_size: int = 16,
|
||||
) -> None:
|
||||
"""
|
||||
Token-parallel implementation: each token gets its own thread block.
|
||||
Supports both fixed-length and variable-length sequences.
|
||||
Reduces wasted computation on padding.
|
||||
|
||||
Writes directly to Aqk and Akk tensors (in-place).
|
||||
|
||||
Args:
|
||||
q: [B, T, H, K]
|
||||
k: [B, T, H, K]
|
||||
gk: [B, T, HV, K] cumsum of gates (HV >= H for GVA)
|
||||
beta: [B, T, HV]
|
||||
Aqk: [B, T, HV, BT] output tensor to write to
|
||||
Akk: [B, T, HV, BC] output tensor for diagonal blocks (fp32)
|
||||
scale: attention scale
|
||||
chunk_size: BT (default 64)
|
||||
sub_chunk_size: BC (default 16)
|
||||
"""
|
||||
B, T, H, K, HV = *q.shape, gk.shape[2]
|
||||
N = len(cu_seqlens) - 1 if cu_seqlens is not None else B
|
||||
BT = chunk_size
|
||||
BC = sub_chunk_size
|
||||
|
||||
def grid(meta):
|
||||
return (B * T, triton.cdiv(HV, meta["BH"]))
|
||||
|
||||
chunk_kda_fwd_kernel_intra_token_parallel[grid](
|
||||
q=q,
|
||||
k=k,
|
||||
g=gk,
|
||||
beta=beta,
|
||||
Aqk=Aqk,
|
||||
Akk=Akk,
|
||||
scale=scale,
|
||||
cu_seqlens=cu_seqlens,
|
||||
N=N,
|
||||
T=T,
|
||||
H=H,
|
||||
HV=HV,
|
||||
K=K,
|
||||
BT=BT,
|
||||
BC=BC,
|
||||
)
|
||||
return Aqk, Akk
|
||||
@@ -0,0 +1,621 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# SPDX-FileCopyrightText: Songlin Yang, Yu Zhang, Zhiyuan Li
|
||||
#
|
||||
# This file contains code adapted from the flash-linear-attention project.
|
||||
# The original source was licensed under the MIT license.
|
||||
# Copyright (c) 2023-2026, Songlin Yang, Yu Zhang, Zhiyuan Li
|
||||
# ruff: noqa: E501
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.third_party.flash_linear_attention.ops.op import exp, log
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.utils.math_utils import cdiv, next_power_of_2
|
||||
|
||||
|
||||
@triton.heuristics(
|
||||
{
|
||||
"HAS_DT_BIAS": lambda args: args["dt_bias"] is not None,
|
||||
"USE_LOWER_BOUND": lambda args: args["lower_bound"] is not None,
|
||||
}
|
||||
)
|
||||
@triton.jit
|
||||
def _kda_gate_beta_fwd_kernel(
|
||||
raw_g,
|
||||
raw_beta,
|
||||
A_log,
|
||||
dt_bias,
|
||||
gate,
|
||||
beta_out,
|
||||
lower_bound,
|
||||
softplus_beta: tl.constexpr,
|
||||
softplus_threshold: tl.constexpr,
|
||||
T,
|
||||
stride_g_token: tl.constexpr,
|
||||
stride_beta_token: tl.constexpr,
|
||||
H: tl.constexpr,
|
||||
D: tl.constexpr,
|
||||
BT: tl.constexpr,
|
||||
BD: tl.constexpr,
|
||||
HAS_DT_BIAS: tl.constexpr,
|
||||
USE_LOWER_BOUND: tl.constexpr,
|
||||
):
|
||||
i_t, i_h = tl.program_id(0), tl.program_id(1)
|
||||
o_t = i_t * BT + tl.arange(0, BT)
|
||||
o_d = tl.arange(0, BD)
|
||||
m_t = o_t < T
|
||||
m_d = o_d < D
|
||||
|
||||
p_g = raw_g + o_t[:, None] * stride_g_token + i_h * D + o_d[None, :]
|
||||
b_g = tl.load(p_g, mask=m_t[:, None] & m_d[None, :], other=0.0).to(tl.float32)
|
||||
if HAS_DT_BIAS:
|
||||
b_bias = tl.load(
|
||||
dt_bias + i_h * D + o_d,
|
||||
mask=m_d,
|
||||
other=0.0,
|
||||
).to(tl.float32)
|
||||
b_g += b_bias[None, :]
|
||||
|
||||
b_a = exp(tl.load(A_log + i_h).to(tl.float32))
|
||||
if USE_LOWER_BOUND:
|
||||
b_gate = lower_bound * tl.sigmoid(b_a * b_g)
|
||||
else:
|
||||
b_scaled = b_g * softplus_beta
|
||||
b_softplus = tl.where(
|
||||
b_scaled > softplus_threshold,
|
||||
b_g,
|
||||
log(1.0 + tl.exp(b_scaled)) / softplus_beta,
|
||||
)
|
||||
b_gate = -b_a * b_softplus
|
||||
|
||||
p_gate = gate + (o_t[:, None] * H + i_h) * D + o_d[None, :]
|
||||
tl.store(
|
||||
p_gate,
|
||||
b_gate,
|
||||
mask=m_t[:, None] & m_d[None, :],
|
||||
)
|
||||
|
||||
b_beta = tl.load(
|
||||
raw_beta + o_t * stride_beta_token + i_h,
|
||||
mask=m_t,
|
||||
other=0.0,
|
||||
).to(tl.float32)
|
||||
tl.store(beta_out + o_t * H + i_h, tl.sigmoid(b_beta), mask=m_t)
|
||||
|
||||
|
||||
def _fused_kda_gate_beta(
|
||||
raw_g: torch.Tensor,
|
||||
raw_beta: torch.Tensor,
|
||||
A_log: torch.Tensor,
|
||||
dt_bias: torch.Tensor | None,
|
||||
lower_bound: float | None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
B, T, H, D = raw_g.shape
|
||||
assert B == 1
|
||||
assert raw_beta.shape == (B, T, H)
|
||||
assert raw_g.stride()[2:] == (D, 1)
|
||||
assert raw_beta.stride(2) == 1
|
||||
gate = torch.empty((B, T, H, D), dtype=torch.float32, device=raw_g.device)
|
||||
beta = torch.empty((B, T, H), dtype=torch.float32, device=raw_beta.device)
|
||||
|
||||
BT = 16
|
||||
_kda_gate_beta_fwd_kernel[(cdiv(T, BT), H)](
|
||||
raw_g=raw_g,
|
||||
raw_beta=raw_beta,
|
||||
A_log=A_log,
|
||||
dt_bias=dt_bias,
|
||||
gate=gate,
|
||||
beta_out=beta,
|
||||
lower_bound=lower_bound,
|
||||
softplus_beta=1.0,
|
||||
softplus_threshold=20.0,
|
||||
T=T,
|
||||
stride_g_token=raw_g.stride(1),
|
||||
stride_beta_token=raw_beta.stride(1),
|
||||
H=H,
|
||||
D=D,
|
||||
BT=BT,
|
||||
BD=next_power_of_2(D),
|
||||
num_warps=4,
|
||||
)
|
||||
return gate, beta
|
||||
|
||||
|
||||
@triton.heuristics(
|
||||
{
|
||||
"IS_SPEC_DECODING": lambda args: args["num_accepted_tokens"] is not None,
|
||||
"HAS_DT_BIAS": lambda args: args["dt_bias"] is not None,
|
||||
"USE_LOWER_BOUND": lambda args: args["lower_bound"] is not None,
|
||||
}
|
||||
)
|
||||
@triton.jit(do_not_specialize=["N", "T"])
|
||||
def fused_recurrent_kda_fwd_kernel(
|
||||
q,
|
||||
k,
|
||||
v,
|
||||
g,
|
||||
beta,
|
||||
A_log,
|
||||
dt_bias,
|
||||
out,
|
||||
state,
|
||||
cu_seqlens,
|
||||
state_indices,
|
||||
num_accepted_tokens,
|
||||
lower_bound,
|
||||
scale: tl.constexpr,
|
||||
N: tl.int64,
|
||||
T: tl.int64,
|
||||
H: tl.constexpr,
|
||||
K: tl.constexpr,
|
||||
V: tl.constexpr,
|
||||
BK: tl.constexpr,
|
||||
BV: tl.constexpr,
|
||||
stride_qkv_token: tl.constexpr,
|
||||
stride_g_token: tl.constexpr,
|
||||
stride_beta_token: tl.constexpr,
|
||||
stride_out_token: tl.constexpr,
|
||||
stride_state_token: tl.constexpr,
|
||||
stride_indices_seq: tl.constexpr,
|
||||
IS_SPEC_DECODING: tl.constexpr,
|
||||
USE_QK_L2NORM_IN_KERNEL: tl.constexpr,
|
||||
USE_GATE_IN_KERNEL: tl.constexpr,
|
||||
APPLY_BETA_SIGMOID: tl.constexpr,
|
||||
HAS_DT_BIAS: tl.constexpr,
|
||||
USE_LOWER_BOUND: tl.constexpr,
|
||||
num_stages: tl.constexpr,
|
||||
):
|
||||
pid = tl.program_id(0)
|
||||
i_v = pid % tl.cdiv(V, BV)
|
||||
i_nh = pid // tl.cdiv(V, BV)
|
||||
i_n, i_h = i_nh // H, i_nh % H
|
||||
bos = tl.load(cu_seqlens + i_n).to(tl.int64)
|
||||
eos = tl.load(cu_seqlens + i_n + 1).to(tl.int64)
|
||||
sequence_length = eos - bos
|
||||
if sequence_length == 0:
|
||||
return
|
||||
|
||||
o_k = tl.arange(0, BK)
|
||||
o_v = i_v * BV + tl.arange(0, BV)
|
||||
m_k = o_k < K
|
||||
m_v = o_v < V
|
||||
m_state = m_v[:, None] & m_k[None, :]
|
||||
|
||||
if IS_SPEC_DECODING:
|
||||
initial_token = tl.load(num_accepted_tokens + i_n).to(tl.int64) - 1
|
||||
else:
|
||||
initial_token = 0
|
||||
state_index = tl.load(state_indices + i_n * stride_indices_seq + initial_token).to(
|
||||
tl.int64
|
||||
)
|
||||
p_out = out + bos * stride_out_token + i_h * V + o_v
|
||||
if state_index <= 0:
|
||||
tl.store(p_out, tl.zeros([BV], dtype=tl.float32), mask=m_v)
|
||||
return
|
||||
|
||||
p_state = (
|
||||
state
|
||||
+ state_index * stride_state_token
|
||||
+ i_h * V * K
|
||||
+ o_v[:, None] * K
|
||||
+ o_k[None, :]
|
||||
)
|
||||
b_state = tl.load(p_state, mask=m_state, other=0.0).to(tl.float32)
|
||||
|
||||
p_q = q + bos * stride_qkv_token + i_h * K + o_k
|
||||
p_k = k + bos * stride_qkv_token + i_h * K + o_k
|
||||
p_v = v + bos * stride_qkv_token + i_h * V + o_v
|
||||
p_g = g + bos * stride_g_token + i_h * K + o_k
|
||||
p_beta = beta + bos * stride_beta_token + i_h
|
||||
for i_t in tl.range(0, sequence_length, num_stages=num_stages):
|
||||
b_q = tl.load(p_q, mask=m_k, other=0.0, eviction_policy="evict_last").to(
|
||||
tl.float32
|
||||
)
|
||||
b_k = tl.load(p_k, mask=m_k, other=0.0, eviction_policy="evict_last").to(
|
||||
tl.float32
|
||||
)
|
||||
b_v = tl.load(p_v, mask=m_v, other=0.0, eviction_policy="evict_first").to(
|
||||
tl.float32
|
||||
)
|
||||
if USE_QK_L2NORM_IN_KERNEL:
|
||||
b_q = b_q / tl.sqrt(tl.sum(b_q * b_q) + 1e-6)
|
||||
b_k = b_k / tl.sqrt(tl.sum(b_k * b_k) + 1e-6)
|
||||
b_q *= scale
|
||||
|
||||
b_gate = tl.load(
|
||||
p_g,
|
||||
mask=m_k,
|
||||
other=0.0,
|
||||
eviction_policy="evict_last",
|
||||
).to(tl.float32)
|
||||
if USE_GATE_IN_KERNEL:
|
||||
if HAS_DT_BIAS:
|
||||
b_bias = tl.load(
|
||||
dt_bias + i_h * K + o_k,
|
||||
mask=m_k,
|
||||
other=0.0,
|
||||
).to(tl.float32)
|
||||
b_gate += b_bias
|
||||
b_a = exp(tl.load(A_log + i_h).to(tl.float32))
|
||||
if USE_LOWER_BOUND:
|
||||
b_gate = lower_bound * tl.sigmoid(b_a * b_gate)
|
||||
else:
|
||||
b_softplus = tl.where(
|
||||
b_gate > 20.0,
|
||||
b_gate,
|
||||
log(1.0 + tl.exp(b_gate)),
|
||||
)
|
||||
b_gate = -b_a * b_softplus
|
||||
|
||||
b_state *= exp(b_gate[None, :])
|
||||
b_v -= tl.sum(b_state * b_k[None, :], axis=1)
|
||||
b_beta = tl.load(p_beta, eviction_policy="evict_last").to(tl.float32)
|
||||
if APPLY_BETA_SIGMOID:
|
||||
b_beta = tl.sigmoid(b_beta)
|
||||
b_v *= b_beta
|
||||
b_state += b_v[:, None] * b_k[None, :]
|
||||
b_out = tl.sum(b_state * b_q[None, :], axis=1)
|
||||
tl.store(
|
||||
p_out,
|
||||
b_out.to(p_out.dtype.element_ty),
|
||||
mask=m_v,
|
||||
eviction_policy="evict_first",
|
||||
)
|
||||
|
||||
final_state_index = tl.load(state_indices + i_n * stride_indices_seq + i_t).to(
|
||||
tl.int64
|
||||
)
|
||||
if final_state_index > 0:
|
||||
p_final_state = (
|
||||
state
|
||||
+ final_state_index * stride_state_token
|
||||
+ i_h * V * K
|
||||
+ o_v[:, None] * K
|
||||
+ o_k[None, :]
|
||||
)
|
||||
tl.store(
|
||||
p_final_state,
|
||||
b_state.to(p_final_state.dtype.element_ty),
|
||||
mask=m_state,
|
||||
)
|
||||
|
||||
p_q += stride_qkv_token
|
||||
p_k += stride_qkv_token
|
||||
p_v += stride_qkv_token
|
||||
p_g += stride_g_token
|
||||
p_beta += stride_beta_token
|
||||
p_out += stride_out_token
|
||||
|
||||
|
||||
def fused_recurrent_kda_fwd(
|
||||
q: torch.Tensor,
|
||||
k: torch.Tensor,
|
||||
v: torch.Tensor,
|
||||
g: torch.Tensor,
|
||||
beta: torch.Tensor,
|
||||
scale: float | None = None,
|
||||
initial_state: torch.Tensor | None = None,
|
||||
inplace_final_state: bool = True,
|
||||
cu_seqlens: torch.Tensor | None = None,
|
||||
ssm_state_indices: torch.Tensor | None = None,
|
||||
num_accepted_tokens: torch.Tensor | None = None,
|
||||
use_qk_l2norm_in_kernel: bool = True,
|
||||
A_log: torch.Tensor | None = None,
|
||||
dt_bias: torch.Tensor | None = None,
|
||||
lower_bound: float | None = None,
|
||||
use_gate_in_kernel: bool = False,
|
||||
use_beta_sigmoid_in_kernel: bool = False,
|
||||
out: torch.Tensor | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Launch recurrent KDA with dense inner dimensions and row strides."""
|
||||
B, T, H, K = q.shape
|
||||
V = v.shape[-1]
|
||||
assert B == 1 and k.shape == q.shape
|
||||
assert v.shape == (B, T, H, V) and g.shape == (B, T, H, K)
|
||||
assert beta.shape == (B, T, H)
|
||||
assert initial_state is not None
|
||||
assert cu_seqlens is not None
|
||||
assert ssm_state_indices is not None
|
||||
assert inplace_final_state
|
||||
if out is None:
|
||||
out = torch.empty_like(v)
|
||||
assert out.shape == v.shape
|
||||
assert initial_state.shape[1:] == (H, V, K)
|
||||
assert ssm_state_indices.ndim in (1, 2)
|
||||
|
||||
assert q.stride()[2:] == k.stride()[2:] == (K, 1)
|
||||
assert v.stride()[2:] == out.stride()[2:] == (V, 1)
|
||||
assert g.stride()[2:] == (K, 1)
|
||||
assert beta.stride(2) == 1
|
||||
assert q.stride(1) == k.stride(1) == v.stride(1)
|
||||
assert initial_state.stride()[1:] == (V * K, K, 1)
|
||||
N = cu_seqlens.numel() - 1
|
||||
if ssm_state_indices.ndim == 1:
|
||||
assert T == N
|
||||
assert num_accepted_tokens is None
|
||||
else:
|
||||
assert ssm_state_indices.stride(1) == 1
|
||||
assert cu_seqlens.is_contiguous()
|
||||
if use_gate_in_kernel:
|
||||
assert A_log is not None and A_log.is_contiguous()
|
||||
assert dt_bias is None or dt_bias.is_contiguous()
|
||||
|
||||
if scale is None:
|
||||
scale = K**-0.5
|
||||
|
||||
BV = 32 if use_gate_in_kernel else 8
|
||||
num_warps = 4 if use_gate_in_kernel else 1
|
||||
grid = (cdiv(V, BV) * N * H,)
|
||||
fused_recurrent_kda_fwd_kernel[grid](
|
||||
q=q,
|
||||
k=k,
|
||||
v=v,
|
||||
g=g,
|
||||
beta=beta,
|
||||
A_log=A_log,
|
||||
dt_bias=dt_bias,
|
||||
out=out,
|
||||
state=initial_state,
|
||||
cu_seqlens=cu_seqlens,
|
||||
state_indices=ssm_state_indices,
|
||||
num_accepted_tokens=num_accepted_tokens,
|
||||
lower_bound=lower_bound,
|
||||
scale=scale,
|
||||
N=N,
|
||||
T=T,
|
||||
H=H,
|
||||
K=K,
|
||||
V=V,
|
||||
BK=next_power_of_2(K),
|
||||
BV=BV,
|
||||
stride_qkv_token=q.stride(1),
|
||||
stride_g_token=g.stride(1),
|
||||
stride_beta_token=beta.stride(1),
|
||||
stride_out_token=out.stride(1),
|
||||
stride_state_token=initial_state.stride(0),
|
||||
stride_indices_seq=ssm_state_indices.stride(0),
|
||||
IS_SPEC_DECODING=num_accepted_tokens is not None,
|
||||
USE_QK_L2NORM_IN_KERNEL=use_qk_l2norm_in_kernel,
|
||||
USE_GATE_IN_KERNEL=use_gate_in_kernel,
|
||||
APPLY_BETA_SIGMOID=use_beta_sigmoid_in_kernel,
|
||||
num_warps=num_warps,
|
||||
num_stages=2,
|
||||
)
|
||||
return out, initial_state
|
||||
|
||||
|
||||
def fused_recurrent_kda(
|
||||
q: torch.Tensor,
|
||||
k: torch.Tensor,
|
||||
v: torch.Tensor,
|
||||
raw_g: torch.Tensor,
|
||||
raw_beta: torch.Tensor,
|
||||
A_log: torch.Tensor,
|
||||
dt_bias: torch.Tensor | None,
|
||||
lower_bound: float | None,
|
||||
initial_state: torch.Tensor,
|
||||
cu_seqlens: torch.Tensor,
|
||||
ssm_state_indices: torch.Tensor,
|
||||
num_accepted_tokens: torch.Tensor | None = None,
|
||||
out: torch.Tensor | None = None,
|
||||
fuse_gate: bool | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Run recurrent KDA from raw gate and beta inputs.
|
||||
|
||||
This vLLM wrapper applies the gate activation and beta sigmoid, selecting
|
||||
whether to materialize them before launching the recurrent kernel.
|
||||
"""
|
||||
if fuse_gate is None:
|
||||
# gfx950: always fuse the gate.
|
||||
fuse_gate = True
|
||||
|
||||
if fuse_gate:
|
||||
gate = raw_g
|
||||
beta = raw_beta
|
||||
else:
|
||||
gate, beta = _fused_kda_gate_beta(
|
||||
raw_g,
|
||||
raw_beta,
|
||||
A_log,
|
||||
dt_bias,
|
||||
lower_bound,
|
||||
)
|
||||
return fused_recurrent_kda_fwd(
|
||||
q=q,
|
||||
k=k,
|
||||
v=v,
|
||||
g=gate,
|
||||
beta=beta,
|
||||
scale=q.shape[-1] ** -0.5,
|
||||
initial_state=initial_state,
|
||||
inplace_final_state=True,
|
||||
cu_seqlens=cu_seqlens,
|
||||
ssm_state_indices=ssm_state_indices,
|
||||
num_accepted_tokens=num_accepted_tokens,
|
||||
use_qk_l2norm_in_kernel=True,
|
||||
A_log=A_log if fuse_gate else None,
|
||||
dt_bias=dt_bias if fuse_gate else None,
|
||||
lower_bound=lower_bound if fuse_gate else None,
|
||||
use_gate_in_kernel=fuse_gate,
|
||||
use_beta_sigmoid_in_kernel=fuse_gate,
|
||||
out=out,
|
||||
)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def fused_recurrent_kda_packed_decode_kernel(
|
||||
mixed_qkv,
|
||||
raw_g,
|
||||
raw_beta,
|
||||
A_log,
|
||||
dt_bias,
|
||||
out,
|
||||
state,
|
||||
state_indices,
|
||||
lower_bound,
|
||||
scale: tl.constexpr,
|
||||
stride_mixed_token: tl.constexpr,
|
||||
stride_g_token: tl.constexpr,
|
||||
stride_beta_token: tl.constexpr,
|
||||
stride_state_token: tl.constexpr,
|
||||
H: tl.constexpr,
|
||||
K: tl.constexpr,
|
||||
V: tl.constexpr,
|
||||
BK: tl.constexpr,
|
||||
BV: tl.constexpr,
|
||||
SOFTPLUS_THRESHOLD: tl.constexpr,
|
||||
USE_LOWER_BOUND: tl.constexpr,
|
||||
):
|
||||
i_v, i_nh = tl.program_id(0), tl.program_id(1)
|
||||
i_n, i_h = i_nh // H, i_nh % H
|
||||
|
||||
o_k = tl.arange(0, BK)
|
||||
o_v = i_v * BV + tl.arange(0, BV)
|
||||
mask_k = o_k < K
|
||||
mask_v = o_v < V
|
||||
mask_state = mask_v[:, None] & mask_k[None, :]
|
||||
|
||||
state_idx = tl.load(state_indices + i_n).to(tl.int64)
|
||||
p_out = out + (i_n * H + i_h) * V + o_v
|
||||
if state_idx <= 0:
|
||||
tl.store(p_out, tl.zeros([BV], dtype=tl.float32), mask=mask_v)
|
||||
return
|
||||
|
||||
p_state = state + state_idx * stride_state_token
|
||||
p_state += i_h * V * K + o_v[:, None] * K + o_k[None, :]
|
||||
b_state = tl.load(p_state, mask=mask_state, other=0).to(tl.float32)
|
||||
|
||||
# Q, K, and V occupy consecutive channel ranges, while the token stride
|
||||
# may also include the output-gate projection that follows packed QKV.
|
||||
p_mixed = mixed_qkv + i_n * stride_mixed_token
|
||||
b_q = tl.load(p_mixed + i_h * K + o_k, mask=mask_k, other=0).to(tl.float32)
|
||||
b_k = tl.load(
|
||||
p_mixed + H * K + i_h * K + o_k,
|
||||
mask=mask_k,
|
||||
other=0,
|
||||
).to(tl.float32)
|
||||
b_v = tl.load(
|
||||
p_mixed + 2 * H * K + i_h * V + o_v,
|
||||
mask=mask_v,
|
||||
other=0,
|
||||
).to(tl.float32)
|
||||
|
||||
b_q /= tl.sqrt(tl.sum(b_q * b_q) + 1e-6)
|
||||
b_k /= tl.sqrt(tl.sum(b_k * b_k) + 1e-6)
|
||||
b_q *= scale
|
||||
|
||||
p_g = raw_g + i_n * stride_g_token + i_h * K + o_k
|
||||
b_g = tl.load(p_g, mask=mask_k, other=0).to(tl.float32)
|
||||
b_bias = tl.load(dt_bias + i_h * K + o_k, mask=mask_k, other=0).to(tl.float32)
|
||||
b_a = exp(tl.load(A_log + i_h).to(tl.float32))
|
||||
b_g += b_bias
|
||||
if USE_LOWER_BOUND:
|
||||
b_gate = lower_bound * tl.sigmoid(b_a * b_g)
|
||||
else:
|
||||
b_softplus = tl.where(
|
||||
b_g > SOFTPLUS_THRESHOLD,
|
||||
b_g,
|
||||
log(1.0 + tl.exp(b_g)),
|
||||
)
|
||||
b_gate = -b_a * b_softplus
|
||||
|
||||
b_state *= exp(b_gate[None, :])
|
||||
b_v -= tl.sum(b_state * b_k[None, :], axis=1)
|
||||
b_beta = tl.sigmoid(
|
||||
tl.load(raw_beta + i_n * stride_beta_token + i_h).to(tl.float32)
|
||||
)
|
||||
b_v *= b_beta
|
||||
b_state += b_v[:, None] * b_k[None, :]
|
||||
b_out = tl.sum(b_state * b_q[None, :], axis=1)
|
||||
|
||||
tl.store(p_out, b_out.to(p_out.dtype.element_ty), mask=mask_v)
|
||||
tl.store(p_state, b_state.to(p_state.dtype.element_ty), mask=mask_state)
|
||||
|
||||
|
||||
def fused_recurrent_kda_packed_decode(
|
||||
mixed_qkv: torch.Tensor,
|
||||
raw_g: torch.Tensor,
|
||||
raw_beta: torch.Tensor,
|
||||
A_log: torch.Tensor,
|
||||
dt_bias: torch.Tensor,
|
||||
lower_bound: float | None,
|
||||
initial_state: torch.Tensor,
|
||||
state_indices: torch.Tensor,
|
||||
scale: float | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Run one-token KDA decode directly from packed post-conv QKV."""
|
||||
if mixed_qkv.ndim != 2 or mixed_qkv.stride(-1) != 1:
|
||||
raise ValueError("`mixed_qkv` must be 2D and contiguous in its last dim.")
|
||||
if raw_g.ndim != 4 or raw_g.shape[0] != 1:
|
||||
raise ValueError("`raw_g` must have shape [1, B, H, K].")
|
||||
if raw_beta.ndim != 3 or raw_beta.shape[0] != 1:
|
||||
raise ValueError("`raw_beta` must have shape [1, B, H].")
|
||||
if initial_state.ndim != 4:
|
||||
raise ValueError("`initial_state` must have shape [cache, H, V, K].")
|
||||
_, H, V, K = initial_state.shape
|
||||
if raw_g.stride()[2:] != (K, 1):
|
||||
raise ValueError("`raw_g` must be contiguous within each token.")
|
||||
if raw_beta.stride(2) != 1:
|
||||
raise ValueError("`raw_beta` heads must be contiguous.")
|
||||
if initial_state.stride()[1:] != (V * K, K, 1):
|
||||
raise ValueError("`initial_state` must be contiguous within each cache slot.")
|
||||
if state_indices.ndim != 1 or state_indices.stride(0) != 1:
|
||||
raise ValueError("`state_indices` must be contiguous and one-dimensional.")
|
||||
if A_log.ndim != 1 or not A_log.is_contiguous():
|
||||
raise ValueError("`A_log` must be contiguous and one-dimensional.")
|
||||
if not dt_bias.is_contiguous():
|
||||
raise ValueError("`dt_bias` must be contiguous.")
|
||||
|
||||
device = mixed_qkv.device
|
||||
if any(
|
||||
x.device != device
|
||||
for x in (raw_g, raw_beta, A_log, dt_bias, initial_state, state_indices)
|
||||
):
|
||||
raise ValueError("All packed KDA inputs must be on the same device.")
|
||||
|
||||
B = mixed_qkv.shape[0]
|
||||
if raw_g.shape != (1, B, H, K):
|
||||
raise ValueError(f"Unexpected raw gate shape {tuple(raw_g.shape)}.")
|
||||
if raw_beta.shape != (1, B, H):
|
||||
raise ValueError(f"Unexpected raw beta shape {tuple(raw_beta.shape)}.")
|
||||
if mixed_qkv.shape[1] != 2 * H * K + H * V:
|
||||
raise ValueError(f"Unexpected packed QKV shape {tuple(mixed_qkv.shape)}.")
|
||||
if A_log.numel() != H or dt_bias.numel() != H * K:
|
||||
raise ValueError("`A_log` or `dt_bias` has an incompatible shape.")
|
||||
if state_indices.shape[0] != B:
|
||||
raise ValueError("`state_indices` must contain one entry per token.")
|
||||
|
||||
BK = next_power_of_2(K)
|
||||
BV = min(next_power_of_2(V), 32)
|
||||
if scale is None:
|
||||
scale = K**-0.5
|
||||
|
||||
out = torch.empty((1, B, H, V), dtype=mixed_qkv.dtype, device=device)
|
||||
grid = (cdiv(V, BV), B * H)
|
||||
fused_recurrent_kda_packed_decode_kernel[grid](
|
||||
mixed_qkv=mixed_qkv,
|
||||
raw_g=raw_g,
|
||||
raw_beta=raw_beta,
|
||||
A_log=A_log,
|
||||
dt_bias=dt_bias,
|
||||
out=out,
|
||||
state=initial_state,
|
||||
state_indices=state_indices,
|
||||
lower_bound=lower_bound or 0.0,
|
||||
scale=scale,
|
||||
stride_mixed_token=mixed_qkv.stride(0),
|
||||
stride_g_token=raw_g.stride(1),
|
||||
stride_beta_token=raw_beta.stride(1),
|
||||
stride_state_token=initial_state.stride(0),
|
||||
H=H,
|
||||
K=K,
|
||||
V=V,
|
||||
BK=BK,
|
||||
BV=BV,
|
||||
SOFTPLUS_THRESHOLD=20.0,
|
||||
USE_LOWER_BOUND=lower_bound is not None,
|
||||
num_warps=4,
|
||||
num_stages=2,
|
||||
)
|
||||
return out, initial_state
|
||||
@@ -0,0 +1,2 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
@@ -0,0 +1,330 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Shared Kimi-K3 multimodal preprocessing."""
|
||||
|
||||
import math
|
||||
from collections.abc import Mapping, Sequence
|
||||
from typing import Any, cast
|
||||
|
||||
import torch
|
||||
from transformers import BatchFeature
|
||||
|
||||
from vllm.config.multimodal import BaseDummyOptions, ImageDummyOptions
|
||||
from vllm.inputs import MultiModalDataDict
|
||||
from vllm.logger import init_logger
|
||||
from vllm.multimodal.inputs import (
|
||||
MultiModalFieldConfig,
|
||||
MultiModalKwargsItems,
|
||||
)
|
||||
from vllm.multimodal.parse import ImageProcessorItems, ImageSize, MultiModalDataItems
|
||||
from vllm.multimodal.processing import (
|
||||
BaseDummyInputsBuilder,
|
||||
BaseMultiModalProcessor,
|
||||
BaseProcessingInfo,
|
||||
InputProcessingContext,
|
||||
PromptReplacement,
|
||||
PromptUpdate,
|
||||
PromptUpdateDetails,
|
||||
)
|
||||
from vllm.transformers_utils.configs.kimi_k3 import KimiK3Config
|
||||
from vllm.transformers_utils.processor import cached_get_image_processor
|
||||
from vllm.transformers_utils.processors.kimi_k3 import KimiK3Processor
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def navit_resize_image(
|
||||
width: int,
|
||||
height: int,
|
||||
patch_size: int,
|
||||
merge_kernel_size: int,
|
||||
in_patch_limit: int,
|
||||
patch_limit_on_one_side: int,
|
||||
fixed_output_tokens: int | None,
|
||||
):
|
||||
# Apply the patch limits.
|
||||
s1 = math.sqrt(
|
||||
in_patch_limit
|
||||
/ (max(1.0, width // patch_size) * max(1.0, height // patch_size))
|
||||
)
|
||||
s2 = patch_limit_on_one_side * patch_size / width
|
||||
s3 = patch_limit_on_one_side * patch_size / height
|
||||
scale = min(1.0, s1, s2, s3)
|
||||
new_w, new_h = max(1, int(width * scale)), max(1, int(height * scale))
|
||||
new_w = min(new_w, patch_limit_on_one_side * patch_size)
|
||||
new_h = min(new_h, patch_limit_on_one_side * patch_size)
|
||||
|
||||
factor = merge_kernel_size * patch_size
|
||||
|
||||
pad_height = (factor - new_h % factor) % factor
|
||||
pad_width = (factor - new_w % factor) % factor
|
||||
|
||||
if fixed_output_tokens is not None:
|
||||
num_tokens = fixed_output_tokens
|
||||
else:
|
||||
# Calculate new dimensions after padding and patching
|
||||
token_height = (new_h + pad_height) // factor
|
||||
token_width = (new_w + pad_width) // factor
|
||||
|
||||
assert token_height * merge_kernel_size <= patch_limit_on_one_side, (
|
||||
f"token_height {token_height} * merge_kernel_size {merge_kernel_size} > "
|
||||
f"patch_limit_on_one_side {patch_limit_on_one_side}"
|
||||
)
|
||||
assert token_width * merge_kernel_size <= patch_limit_on_one_side, (
|
||||
f"token_width {token_width} * merge_kernel_size {merge_kernel_size} > "
|
||||
f"patch_limit_on_one_side {patch_limit_on_one_side}"
|
||||
)
|
||||
|
||||
num_tokens = token_height * token_width
|
||||
return {
|
||||
"num_tokens": num_tokens,
|
||||
"new_width": new_w,
|
||||
"new_height": new_h,
|
||||
"pad_width": pad_width,
|
||||
"pad_height": pad_height,
|
||||
"sampled_nframes": 1,
|
||||
}
|
||||
|
||||
|
||||
class KimiK3ProcessingInfo(BaseProcessingInfo):
|
||||
"""Processing information for the image-only Kimi-K3 model.
|
||||
|
||||
K3 uses the standard ``image`` modality (unlike K2.5's unified
|
||||
``vision_chunk``), so it builds its own ``KimiK3Processor`` wrapper around
|
||||
the checkpoint's image processor and resolves the ``<|media_pad|>`` token
|
||||
id the same way K2.5 does.
|
||||
"""
|
||||
|
||||
def __init__(self, ctx: InputProcessingContext) -> None:
|
||||
super().__init__(ctx)
|
||||
|
||||
self.hf_config = hf_config = self.get_hf_config()
|
||||
|
||||
tokenizer = self.get_tokenizer()
|
||||
image_processor = cached_get_image_processor(
|
||||
self.ctx.model_config.model,
|
||||
revision=self.ctx.model_config.revision,
|
||||
trust_remote_code=self.ctx.model_config.trust_remote_code,
|
||||
)
|
||||
|
||||
# Resolve token ID from the tokenizer because transformers v5
|
||||
# may remap token IDs vs config.json.
|
||||
config_token_id = hf_config.media_placeholder_token_id
|
||||
resolved_token_id = tokenizer.convert_tokens_to_ids("<|media_pad|>")
|
||||
unk_token_id = getattr(tokenizer, "unk_token_id", None)
|
||||
is_valid_resolved = isinstance(resolved_token_id, int) and (
|
||||
unk_token_id is None or resolved_token_id != unk_token_id
|
||||
)
|
||||
if is_valid_resolved and resolved_token_id != config_token_id:
|
||||
logger.warning_once(
|
||||
"Kimi-K3 config.media_placeholder_token_id (%d) disagrees "
|
||||
"with tokenizer mapping for <|media_pad|> (%d). "
|
||||
"Using tokenizer value.",
|
||||
config_token_id,
|
||||
resolved_token_id,
|
||||
)
|
||||
media_token_id = resolved_token_id
|
||||
# Patch config so downstream code also sees the correct ID.
|
||||
hf_config.media_placeholder_token_id = resolved_token_id
|
||||
else:
|
||||
media_token_id = config_token_id
|
||||
|
||||
self.media_token_id = media_token_id
|
||||
self.media_token = tokenizer.decode(media_token_id)
|
||||
|
||||
self.image_processor = image_processor
|
||||
self.hf_processor = KimiK3Processor(
|
||||
tokenizer=tokenizer,
|
||||
image_processor=image_processor,
|
||||
)
|
||||
self.media_tokens_calculator = image_processor.media_tokens_calculator
|
||||
|
||||
def get_hf_processor(self, **kwargs: object) -> KimiK3Processor:
|
||||
return self.hf_processor
|
||||
|
||||
def get_hf_config(self) -> KimiK3Config:
|
||||
return self.ctx.get_hf_config(KimiK3Config)
|
||||
|
||||
def get_supported_mm_limits(self) -> Mapping[str, int | None]:
|
||||
# None means unlimited
|
||||
return {"image": None}
|
||||
|
||||
@classmethod
|
||||
def get_max_image_size(
|
||||
cls,
|
||||
patch_size: int,
|
||||
merge_kernel_size: int,
|
||||
in_patch_limit: int,
|
||||
patch_limit_on_one_side: int,
|
||||
fixed_output_tokens: int | None,
|
||||
) -> ImageSize:
|
||||
max_side = patch_limit_on_one_side * patch_size
|
||||
best_score = (-1, -1)
|
||||
best_size = (max_side, max_side)
|
||||
|
||||
for width_patches in range(patch_limit_on_one_side + 1):
|
||||
width = min((width_patches + 1) * patch_size - 1, max_side)
|
||||
for height_patches in range(width_patches, patch_limit_on_one_side + 1):
|
||||
height = min((height_patches + 1) * patch_size - 1, max_side)
|
||||
resize_config = navit_resize_image(
|
||||
width,
|
||||
height,
|
||||
patch_size,
|
||||
merge_kernel_size,
|
||||
in_patch_limit,
|
||||
patch_limit_on_one_side,
|
||||
fixed_output_tokens,
|
||||
)
|
||||
padded_width = resize_config["new_width"] + resize_config["pad_width"]
|
||||
padded_height = (
|
||||
resize_config["new_height"] + resize_config["pad_height"]
|
||||
)
|
||||
num_patches = padded_width // patch_size * (padded_height // patch_size)
|
||||
score = (resize_config["num_tokens"], num_patches)
|
||||
if score > best_score:
|
||||
best_score = score
|
||||
best_size = (width, height)
|
||||
return ImageSize(width=best_size[0], height=best_size[1])
|
||||
|
||||
|
||||
class KimiK3DummyInputsBuilder(BaseDummyInputsBuilder[KimiK3ProcessingInfo]):
|
||||
"""Builds image-based dummy inputs for K3 profiling.
|
||||
|
||||
The dummy text is made of ``<|kimi_image_placeholder|>`` tokens — exactly
|
||||
the placeholder that K3's ``_get_prompt_updates`` expands — and the dummy
|
||||
mm data is a plain list of PIL images under the ``image`` key.
|
||||
"""
|
||||
|
||||
def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||
num_images = mm_counts.get("image", 0)
|
||||
return self.info.get_hf_config().image_placeholder * num_images
|
||||
|
||||
def get_dummy_mm_data(
|
||||
self,
|
||||
seq_len: int,
|
||||
mm_counts: Mapping[str, int],
|
||||
mm_options: Mapping[str, BaseDummyOptions] | None = None,
|
||||
) -> MultiModalDataDict:
|
||||
media_proc_cfg = self.info.image_processor.media_proc_cfg
|
||||
max_size = self.info.get_max_image_size(
|
||||
media_proc_cfg["patch_size"],
|
||||
media_proc_cfg["merge_kernel_size"],
|
||||
media_proc_cfg["in_patch_limit"],
|
||||
media_proc_cfg["patch_limit_on_one_side"],
|
||||
media_proc_cfg["fixed_output_tokens"],
|
||||
)
|
||||
num_images = mm_counts.get("image", 0)
|
||||
image_overrides = cast(
|
||||
ImageDummyOptions | None,
|
||||
mm_options.get("image") if mm_options else None,
|
||||
)
|
||||
return {
|
||||
"image": self._get_dummy_images(
|
||||
width=max_size.width,
|
||||
height=max_size.height,
|
||||
num_images=num_images,
|
||||
overrides=image_overrides,
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
class KimiK3MultiModalProcessor(BaseMultiModalProcessor[KimiK3ProcessingInfo]):
|
||||
"""Image-only multi-modal processor for Kimi-K3."""
|
||||
|
||||
def _call_hf_processor(
|
||||
self,
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
tok_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
# Override so the base always routes through the text+mm path
|
||||
# (`KimiK3Processor.__call__`). Otherwise the mm-only fast path calls
|
||||
# the checkpoint image processor directly with bare PIL images, but it
|
||||
# requires `{"type": "image", "image": PIL}` media dicts that only our
|
||||
# wrapper builds.
|
||||
return super()._call_hf_processor(prompt, mm_data, mm_kwargs, tok_kwargs)
|
||||
|
||||
def _hf_processor_applies_updates(
|
||||
self,
|
||||
prompt_text: str,
|
||||
mm_items: MultiModalDataItems,
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
tokenization_kwargs: Mapping[str, object],
|
||||
) -> bool:
|
||||
return False
|
||||
|
||||
def _get_mm_fields_config(
|
||||
self,
|
||||
hf_inputs: BatchFeature,
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
) -> Mapping[str, MultiModalFieldConfig]:
|
||||
"""Slice the flattened patch tensor back into per-image items.
|
||||
|
||||
``pixel_values`` holds all patches from every image concatenated; each
|
||||
image's patch count is ``prod(grid_thws[i])``. ``grid_thws`` is one
|
||||
``[N_t, N_h, N_w]`` row per image.
|
||||
"""
|
||||
grid_thws = hf_inputs.get("grid_thws", torch.empty((0, 3)))
|
||||
grid_sizes = grid_thws.prod(-1)
|
||||
|
||||
return dict(
|
||||
pixel_values=MultiModalFieldConfig.flat_from_sizes("image", grid_sizes),
|
||||
grid_thws=MultiModalFieldConfig.batched("image", keep_on_cpu=True),
|
||||
)
|
||||
|
||||
def _get_prompt_updates(
|
||||
self,
|
||||
mm_items: MultiModalDataItems,
|
||||
hf_processor_mm_kwargs: Mapping[str, Any],
|
||||
out_mm_kwargs: MultiModalKwargsItems,
|
||||
) -> Sequence[PromptUpdate]:
|
||||
"""Expand each K3 image placeholder into a resolution-aware update.
|
||||
|
||||
K3's prompt carries a single ``<|kimi_image_placeholder|>`` token per
|
||||
image. This replaces that token with
|
||||
``<|media_begin|>image {w}x{h}<|media_content|>{pads}<|media_end|>``,
|
||||
embedding the per-image resolution in the prompt and marking only the
|
||||
``<|media_pad|>`` positions as embedding slots (the number of pads is
|
||||
the feature size returned by ``media_tokens_calculator``).
|
||||
"""
|
||||
media_token_id = self.info.media_token_id
|
||||
media_token = self.info.media_token
|
||||
image_placeholder = self.info.get_hf_config().image_placeholder
|
||||
|
||||
def get_replacement(item_idx: int) -> PromptUpdateDetails[str]:
|
||||
images = mm_items.get_items("image", ImageProcessorItems)
|
||||
image = images.get(item_idx)
|
||||
if image is None:
|
||||
raise ValueError(f"Missing image data at index {item_idx}")
|
||||
|
||||
# The checkpoint image processor works on media dicts, so wrap the
|
||||
# PIL image before asking it for the token count.
|
||||
num_media_token = self.info.media_tokens_calculator(
|
||||
{"type": "image", "image": image}
|
||||
)
|
||||
pads = media_token * num_media_token
|
||||
|
||||
# NOTE: `width`/`height` are the ORIGINAL upload dimensions, not the
|
||||
# post-preprocess (smart-resized) ones. `image` comes from the
|
||||
# untouched parsed `mm_items`; the checkpoint image processor
|
||||
# (`KimiK3VisionProcessor.preprocess`) only produces new tensors via
|
||||
# `image.resize(...)` and never mutates the stored PIL. This matches
|
||||
# the reference HF processor (`KimiK3Processor.preprocess_medias`),
|
||||
# which also builds the prompt from the original `img.size`. The
|
||||
# resize is reflected only in the pad count above.
|
||||
width, height = images.get_image_size(item_idx)
|
||||
full = (
|
||||
f"<|media_begin|>image {width}x{height}<|media_content|>"
|
||||
f"{pads}<|media_end|>"
|
||||
)
|
||||
|
||||
return PromptUpdateDetails.select_token_id(full, media_token_id)
|
||||
|
||||
return [
|
||||
PromptReplacement(
|
||||
modality="image",
|
||||
target=image_placeholder,
|
||||
replacement=get_replacement,
|
||||
),
|
||||
]
|
||||
@@ -0,0 +1,96 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Fused Kimi-K3 MTP input preparation."""
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.triton_utils import tl, triton
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _rms_norm(x, weight, eps, hidden_size: tl.constexpr):
|
||||
x = x.to(tl.float32)
|
||||
variance = tl.sum(x * x, axis=0) / hidden_size
|
||||
return x * tl.rsqrt(variance + eps) * weight.to(tl.float32)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _fused_mtp_input_kernel(
|
||||
positions_ptr,
|
||||
inputs_embeds_ptr,
|
||||
previous_hidden_states_ptr,
|
||||
enorm_weight_ptr,
|
||||
hnorm_weight_ptr,
|
||||
output_ptr,
|
||||
eps,
|
||||
inputs_embeds_stride,
|
||||
previous_hidden_states_stride,
|
||||
output_stride,
|
||||
hidden_size: tl.constexpr,
|
||||
block_size: tl.constexpr,
|
||||
):
|
||||
token_idx = tl.program_id(0).to(tl.int64)
|
||||
input_idx = tl.program_id(1)
|
||||
offsets = tl.arange(0, block_size)
|
||||
mask = offsets < hidden_size
|
||||
|
||||
if input_idx == 0:
|
||||
position = tl.load(positions_ptr + token_idx)
|
||||
values = tl.load(
|
||||
inputs_embeds_ptr + token_idx * inputs_embeds_stride + offsets,
|
||||
mask=mask & (position != 0),
|
||||
other=0.0,
|
||||
)
|
||||
weight = tl.load(enorm_weight_ptr + offsets, mask=mask, other=0.0)
|
||||
else:
|
||||
values = tl.load(
|
||||
previous_hidden_states_ptr
|
||||
+ token_idx * previous_hidden_states_stride
|
||||
+ offsets,
|
||||
mask=mask,
|
||||
other=0.0,
|
||||
)
|
||||
weight = tl.load(hnorm_weight_ptr + offsets, mask=mask, other=0.0)
|
||||
|
||||
output = _rms_norm(values, weight, eps, hidden_size)
|
||||
tl.store(
|
||||
output_ptr + token_idx * output_stride + input_idx * hidden_size + offsets,
|
||||
output,
|
||||
mask=mask,
|
||||
)
|
||||
|
||||
|
||||
def fused_mtp_input(
|
||||
positions: torch.Tensor,
|
||||
inputs_embeds: torch.Tensor,
|
||||
previous_hidden_states: torch.Tensor,
|
||||
enorm_weight: torch.Tensor,
|
||||
hnorm_weight: torch.Tensor,
|
||||
eps: float,
|
||||
) -> torch.Tensor:
|
||||
"""Mask and normalize both MTP inputs into the projection layout."""
|
||||
num_tokens, hidden_size = inputs_embeds.shape
|
||||
output = torch.empty(
|
||||
num_tokens,
|
||||
2 * hidden_size,
|
||||
dtype=inputs_embeds.dtype,
|
||||
device=inputs_embeds.device,
|
||||
)
|
||||
if num_tokens == 0:
|
||||
return output
|
||||
|
||||
_fused_mtp_input_kernel[(num_tokens, 2)](
|
||||
positions,
|
||||
inputs_embeds,
|
||||
previous_hidden_states,
|
||||
enorm_weight,
|
||||
hnorm_weight,
|
||||
output,
|
||||
eps,
|
||||
inputs_embeds.stride(0),
|
||||
previous_hidden_states.stride(0),
|
||||
output.stride(0),
|
||||
hidden_size,
|
||||
triton.next_power_of_2(hidden_size),
|
||||
)
|
||||
return output
|
||||
@@ -0,0 +1,527 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""K3 dense MLA draft model for DSpark speculative decoding."""
|
||||
|
||||
from collections.abc import Iterable
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
import torch.nn.functional as F
|
||||
|
||||
import vllm._custom_ops as ops
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.model_executor.layers.linear import ReplicatedLinear
|
||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||
from vllm.model_executor.models.qwen3_dspark import DSparkMarkovHead
|
||||
from vllm.model_executor.models.utils import (
|
||||
AutoWeightsLoader,
|
||||
WeightsMapper,
|
||||
get_draft_quant_config,
|
||||
maybe_prefix,
|
||||
)
|
||||
from vllm.models.kimi_k3.nvidia.mla import MultiHeadLatentAttention
|
||||
from vllm.models.kimi_k3.nvidia.model import KimiMLP
|
||||
from vllm.utils.torch_utils import is_quantized_kv_cache
|
||||
|
||||
|
||||
class ReplicatedDSparkMarkovHead(DSparkMarkovHead):
|
||||
"""DSpark Markov head with full weights on every TP rank."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vocab_size: int,
|
||||
draft_vocab_size: int,
|
||||
markov_rank: int,
|
||||
prefix: str,
|
||||
) -> None:
|
||||
# TODO: Remove this mypy workaround once the K3 PR is fully merged.
|
||||
super().__init__( # type: ignore[call-arg]
|
||||
vocab_size,
|
||||
draft_vocab_size,
|
||||
markov_rank,
|
||||
prefix,
|
||||
replicated=True,
|
||||
)
|
||||
|
||||
|
||||
class K3DSparkDecoderLayer(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
vllm_config: VllmConfig,
|
||||
config,
|
||||
layer_idx: int,
|
||||
start_layer_id: int,
|
||||
prefix: str,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
quant_config = get_draft_quant_config(vllm_config)
|
||||
self.self_attn = MultiHeadLatentAttention(
|
||||
config=config,
|
||||
hidden_size=config.hidden_size,
|
||||
num_heads=config.num_attention_heads,
|
||||
qk_nope_head_dim=config.qk_nope_head_dim,
|
||||
qk_rope_head_dim=config.qk_rope_head_dim,
|
||||
v_head_dim=config.v_head_dim,
|
||||
q_lora_rank=config.q_lora_rank,
|
||||
kv_lora_rank=config.kv_lora_rank,
|
||||
cache_config=vllm_config.cache_config,
|
||||
quant_config=quant_config,
|
||||
prefix=maybe_prefix(
|
||||
prefix, f"layers.{start_layer_id + layer_idx}.self_attn"
|
||||
),
|
||||
use_rope=True,
|
||||
non_causal_multi_token_decode=True,
|
||||
)
|
||||
self.mlp = KimiMLP(
|
||||
hidden_size=config.hidden_size,
|
||||
intermediate_size=config.intermediate_size,
|
||||
hidden_act=config.hidden_act,
|
||||
quant_config=quant_config,
|
||||
prefix=maybe_prefix(prefix, f"layers.{start_layer_id + layer_idx}.mlp"),
|
||||
)
|
||||
self.input_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
self.post_attention_layernorm = RMSNorm(
|
||||
config.hidden_size, eps=config.rms_norm_eps
|
||||
)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
positions: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
residual: torch.Tensor | None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
if residual is None:
|
||||
residual = hidden_states
|
||||
hidden_states = self.input_layernorm(hidden_states)
|
||||
else:
|
||||
hidden_states, residual = self.input_layernorm(hidden_states, residual)
|
||||
|
||||
hidden_states = self.self_attn(
|
||||
positions=positions,
|
||||
hidden_states=hidden_states,
|
||||
)
|
||||
hidden_states, residual = self.post_attention_layernorm(hidden_states, residual)
|
||||
hidden_states = self.mlp(hidden_states)
|
||||
return hidden_states, residual
|
||||
|
||||
|
||||
class K3DSparkModel(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
vllm_config: VllmConfig,
|
||||
start_layer_id: int,
|
||||
prefix: str,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
assert vllm_config.speculative_config is not None
|
||||
self.config = vllm_config.speculative_config.draft_model_config.hf_config
|
||||
self.quant_config = get_draft_quant_config(vllm_config)
|
||||
|
||||
# The frozen target embedding is aliased after the draft checkpoint loads.
|
||||
self.embed_tokens: nn.Module | None = None
|
||||
|
||||
self.context_proj = ReplicatedLinear(
|
||||
self.config.target_hidden_size * self.config.num_target_layers,
|
||||
self.config.hidden_size,
|
||||
bias=False,
|
||||
return_bias=False,
|
||||
quant_config=self.quant_config,
|
||||
prefix=maybe_prefix(prefix, "context_proj"),
|
||||
)
|
||||
self.context_norm = RMSNorm(
|
||||
self.config.hidden_size, eps=self.config.rms_norm_eps
|
||||
)
|
||||
|
||||
self.layers = nn.ModuleList(
|
||||
[
|
||||
K3DSparkDecoderLayer(
|
||||
vllm_config=vllm_config,
|
||||
config=self.config,
|
||||
layer_idx=layer_idx,
|
||||
start_layer_id=start_layer_id,
|
||||
prefix=prefix,
|
||||
)
|
||||
for layer_idx in range(self.config.num_hidden_layers)
|
||||
]
|
||||
)
|
||||
self.final_norm = RMSNorm(self.config.hidden_size, eps=self.config.rms_norm_eps)
|
||||
self.markov_head = ReplicatedDSparkMarkovHead(
|
||||
self.config.vocab_size,
|
||||
self.config.draft_vocab_size,
|
||||
self.config.markov_rank,
|
||||
prefix=maybe_prefix(prefix, "markov_head"),
|
||||
)
|
||||
self._context_kv_fusion_available: bool | None = None
|
||||
self._max_num_context_tokens = (
|
||||
vllm_config.scheduler_config.max_num_batched_tokens
|
||||
)
|
||||
|
||||
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
|
||||
assert self.embed_tokens is not None
|
||||
return self.embed_tokens(input_ids)
|
||||
|
||||
def combine_hidden_states(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
||||
return self.context_norm(self.context_proj(hidden_states))
|
||||
|
||||
@torch.inference_mode()
|
||||
def precompute_and_store_context_kv(
|
||||
self,
|
||||
context_states: torch.Tensor,
|
||||
context_positions: torch.Tensor,
|
||||
context_slot_mapping: torch.Tensor | list[torch.Tensor | None] | None = None,
|
||||
) -> None:
|
||||
"""Project target-derived context into each draft layer's latent cache."""
|
||||
if self._context_kv_fusion_available is None:
|
||||
self._build_fused_context_kv_buffers()
|
||||
if self._context_kv_fusion_available:
|
||||
self._precompute_fused_context_kv(
|
||||
context_states, context_positions, context_slot_mapping
|
||||
)
|
||||
return
|
||||
|
||||
# Quantized fallback. Directly invoking the projection modules preserves
|
||||
# their quantization methods, at the cost of also computing unused Q rows.
|
||||
for layer_idx, layer in enumerate(self.layers):
|
||||
attn = layer.self_attn
|
||||
assert attn.fused_qkv_a_proj is not None
|
||||
assert attn.q_lora_rank is not None
|
||||
assert attn.rotary_emb is not None
|
||||
qkv_lora = attn.fused_qkv_a_proj(context_states)[0]
|
||||
kv_lora = qkv_lora[..., attn.q_lora_rank :]
|
||||
kv_c, k_pe = kv_lora.split(
|
||||
[attn.kv_lora_rank, attn.qk_rope_head_dim], dim=-1
|
||||
)
|
||||
kv_c = attn.kv_a_layernorm(kv_c)
|
||||
k_pe = k_pe.unsqueeze(1)
|
||||
# DeepSeek YaRN's FlashInfer path requires paired Q/K tensors.
|
||||
# The vLLM CUDA op supports rotating one tensor in place and
|
||||
# consumes the same (possibly scaled fp32) cos/sin cache.
|
||||
rotary_emb = attn.rotary_emb
|
||||
ops.rotary_embedding(
|
||||
context_positions,
|
||||
k_pe,
|
||||
None,
|
||||
rotary_emb.head_size,
|
||||
rotary_emb.cos_sin_cache,
|
||||
rotary_emb.is_neox_style,
|
||||
)
|
||||
|
||||
slot_mapping = (
|
||||
context_slot_mapping[layer_idx]
|
||||
if isinstance(context_slot_mapping, (list, tuple))
|
||||
else context_slot_mapping
|
||||
)
|
||||
if slot_mapping is None:
|
||||
continue
|
||||
attn.impl.do_kv_cache_update(
|
||||
kv_c,
|
||||
k_pe,
|
||||
attn.kv_cache,
|
||||
slot_mapping,
|
||||
attn.kv_cache_dtype,
|
||||
attn._k_scale,
|
||||
)
|
||||
|
||||
def _build_fused_context_kv_buffers(self) -> None:
|
||||
"""Build a cross-layer KV-only A projection after checkpoint loading."""
|
||||
if self.quant_config is not None:
|
||||
self._context_kv_fusion_available = False
|
||||
return
|
||||
|
||||
attentions = [layer.self_attn for layer in self.layers]
|
||||
if not attentions or any(
|
||||
attn.fused_qkv_a_proj is None
|
||||
or not hasattr(attn.fused_qkv_a_proj, "weight")
|
||||
for attn in attentions
|
||||
):
|
||||
self._context_kv_fusion_available = False
|
||||
return
|
||||
|
||||
attn0 = attentions[0]
|
||||
assert attn0.q_lora_rank is not None
|
||||
kv_width = attn0.kv_lora_rank + attn0.qk_rope_head_dim
|
||||
kv_weights = []
|
||||
for attn in attentions:
|
||||
assert attn.q_lora_rank is not None
|
||||
assert (
|
||||
attn.q_lora_rank == attn0.q_lora_rank
|
||||
and attn.kv_lora_rank == attn0.kv_lora_rank
|
||||
and attn.qk_rope_head_dim == attn0.qk_rope_head_dim
|
||||
and attn.kv_a_layernorm.variance_epsilon
|
||||
== attn0.kv_a_layernorm.variance_epsilon
|
||||
), "All MLA DSpark layers must share their latent KV geometry."
|
||||
kv_weights.append(
|
||||
attn.fused_qkv_a_proj.weight.detach().narrow(
|
||||
0, attn.q_lora_rank, kv_width
|
||||
)
|
||||
)
|
||||
|
||||
# [L * (kv_lora_rank + rope_dim), hidden_size]. The underlying fused
|
||||
# A weights are replicated (`disable_tp=True`), so this is valid on
|
||||
# every TP rank without communication.
|
||||
self._fused_context_kv_weight = torch.cat(kv_weights, dim=0)
|
||||
self._context_kv_norm_weights = torch.stack(
|
||||
[attn.kv_a_layernorm.weight.detach() for attn in attentions], dim=0
|
||||
).contiguous()
|
||||
self._num_context_layers = len(attentions)
|
||||
self._context_kv_width = kv_width
|
||||
self._context_kv_lora_rank = attn0.kv_lora_rank
|
||||
self._context_rope_dim = attn0.qk_rope_head_dim
|
||||
self._context_rms_norm_eps = attn0.kv_a_layernorm.variance_epsilon
|
||||
self._context_positions_repeated = torch.empty(
|
||||
self._num_context_layers * self._max_num_context_tokens,
|
||||
dtype=torch.int64,
|
||||
device=self._fused_context_kv_weight.device,
|
||||
)
|
||||
self._context_kv_fusion_available = True
|
||||
|
||||
def _precompute_fused_context_kv(
|
||||
self,
|
||||
context_states: torch.Tensor,
|
||||
context_positions: torch.Tensor,
|
||||
context_slot_mapping: torch.Tensor | list[torch.Tensor | None] | None,
|
||||
) -> None:
|
||||
num_ctx = context_states.shape[0]
|
||||
num_layers = self._num_context_layers
|
||||
|
||||
# One KV-only GEMM replaces five full Q+KV GEMMs. For K3 this projects
|
||||
# 5*576 rows rather than 5*2112 rows (72.7% fewer A-projection FLOPs).
|
||||
all_kv = F.linear(context_states, self._fused_context_kv_weight)
|
||||
all_kv = all_kv.view(num_ctx, num_layers, self._context_kv_width)
|
||||
all_kv_c = all_kv[..., : self._context_kv_lora_rank]
|
||||
all_k_pe = all_kv[..., self._context_kv_lora_rank :]
|
||||
|
||||
# Layer-major layout lets the 2-D RMSNorm weights select a distinct row
|
||||
# for each draft layer in one grouped kernel.
|
||||
all_kv_c = all_kv_c.permute(1, 0, 2).contiguous()
|
||||
all_kv_c_normed = torch.empty_like(all_kv_c)
|
||||
ops.rms_norm(
|
||||
all_kv_c_normed,
|
||||
all_kv_c,
|
||||
self._context_kv_norm_weights,
|
||||
self._context_rms_norm_eps,
|
||||
)
|
||||
|
||||
all_k_pe = all_k_pe.permute(1, 0, 2).contiguous()
|
||||
all_k_pe_flat = all_k_pe.view(num_layers * num_ctx, 1, self._context_rope_dim)
|
||||
repeated_positions = self._context_positions_repeated[: num_layers * num_ctx]
|
||||
repeated_positions.view(num_layers, num_ctx).copy_(context_positions)
|
||||
# Keep the single-tensor context RoPE on vLLM's optimized CUDA op;
|
||||
# DeepSeek YaRN's FlashInfer wrapper assumes a non-null key tensor.
|
||||
rotary_emb = self.layers[0].self_attn.rotary_emb
|
||||
assert rotary_emb is not None
|
||||
ops.rotary_embedding(
|
||||
repeated_positions,
|
||||
all_k_pe_flat,
|
||||
None,
|
||||
rotary_emb.head_size,
|
||||
rotary_emb.cos_sin_cache,
|
||||
rotary_emb.is_neox_style,
|
||||
)
|
||||
all_k_pe = all_k_pe_flat.view(num_layers, num_ctx, 1, self._context_rope_dim)
|
||||
|
||||
if context_slot_mapping is None:
|
||||
return
|
||||
|
||||
cache_layers = [layer.self_attn for layer in self.layers]
|
||||
if (
|
||||
not is_quantized_kv_cache(cache_layers[0].kv_cache_dtype)
|
||||
and self._has_uniform_block_layout(cache_layers)
|
||||
and (
|
||||
isinstance(context_slot_mapping, torch.Tensor)
|
||||
or all(s is not None for s in context_slot_mapping)
|
||||
)
|
||||
):
|
||||
# Grouped context KV insert only supports unquantized (bf16) KV cache
|
||||
# and assumes that all layers share the same block layout.
|
||||
|
||||
if isinstance(context_slot_mapping, (list, tuple)):
|
||||
per_layer_slot_mappings = [
|
||||
s for s in context_slot_mapping if s is not None
|
||||
]
|
||||
if len({s.data_ptr() for s in per_layer_slot_mappings}) == 1:
|
||||
# All rows alias to the same slot mapping.
|
||||
slot_mapping = (
|
||||
per_layer_slot_mappings[0].unsqueeze(0).expand(num_layers, -1)
|
||||
)
|
||||
else:
|
||||
slot_mapping = torch.stack(per_layer_slot_mappings, dim=0)
|
||||
else:
|
||||
# Broadcast the single shared context_slot_mapping tensor.
|
||||
slot_mapping = context_slot_mapping.unsqueeze(0).expand(num_layers, -1)
|
||||
|
||||
ref_cache = cache_layers[0].kv_cache
|
||||
ops.concat_and_cache_mla_grouped(
|
||||
all_kv_c_normed,
|
||||
all_k_pe.squeeze(2),
|
||||
self._get_context_kv_cache_ptrs(cache_layers),
|
||||
slot_mapping,
|
||||
ref_cache.size(1),
|
||||
ref_cache.stride(0),
|
||||
ref_cache.stride(1),
|
||||
)
|
||||
return
|
||||
|
||||
for layer_idx, layer in enumerate(self.layers):
|
||||
slot_mapping = (
|
||||
context_slot_mapping[layer_idx]
|
||||
if isinstance(context_slot_mapping, (list, tuple))
|
||||
else context_slot_mapping
|
||||
)
|
||||
if slot_mapping is None:
|
||||
continue
|
||||
attn = layer.self_attn
|
||||
attn.impl.do_kv_cache_update(
|
||||
all_kv_c_normed[layer_idx],
|
||||
all_k_pe[layer_idx],
|
||||
attn.kv_cache,
|
||||
slot_mapping,
|
||||
attn.kv_cache_dtype,
|
||||
attn._k_scale,
|
||||
)
|
||||
|
||||
def _has_uniform_block_layout(
|
||||
self,
|
||||
cache_layers: list[MultiHeadLatentAttention],
|
||||
) -> bool:
|
||||
if not hasattr(self, "_layers_share_kv_block_layout"):
|
||||
ref_cache = cache_layers[0].kv_cache
|
||||
self._layers_share_kv_block_layout = all(
|
||||
cl.kv_cache.size(1) == ref_cache.size(1)
|
||||
and cl.kv_cache.stride(0) == ref_cache.stride(0)
|
||||
and cl.kv_cache.stride(1) == ref_cache.stride(1)
|
||||
for cl in cache_layers
|
||||
)
|
||||
return self._layers_share_kv_block_layout
|
||||
|
||||
def _get_context_kv_cache_ptrs(
|
||||
self,
|
||||
cache_layers: list[MultiHeadLatentAttention],
|
||||
) -> torch.Tensor:
|
||||
# The per-layer KV cache base pointers are stable after allocation, so
|
||||
# build the pointer array once and return it on every call.
|
||||
if not hasattr(self, "_context_cache_ptrs"):
|
||||
ref_cache = cache_layers[0].kv_cache
|
||||
cache_ptrs = torch.tensor(
|
||||
[cl.kv_cache.data_ptr() for cl in cache_layers],
|
||||
dtype=torch.int64,
|
||||
device=ref_cache.device,
|
||||
)
|
||||
self._context_cache_ptrs = cache_ptrs
|
||||
return self._context_cache_ptrs
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
inputs_embeds: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
if inputs_embeds is None:
|
||||
inputs_embeds = self.embed_input_ids(input_ids)
|
||||
|
||||
hidden_states = inputs_embeds
|
||||
residual = None
|
||||
for layer in self.layers:
|
||||
hidden_states, residual = layer(
|
||||
positions=positions,
|
||||
hidden_states=hidden_states,
|
||||
residual=residual,
|
||||
)
|
||||
hidden_states, _ = self.final_norm(hidden_states, residual)
|
||||
return hidden_states
|
||||
|
||||
|
||||
class K3DSparkForCausalLM(nn.Module):
|
||||
has_own_embed_tokens = False
|
||||
has_own_lm_head = False
|
||||
draft_id_to_target_id = None
|
||||
checkpoint_skip_substrs = ("confidence_head", "embed_tokens", "lm_head")
|
||||
|
||||
hf_to_vllm_mapper = WeightsMapper(
|
||||
orig_to_new_prefix={"": "model."},
|
||||
orig_to_new_stacked={
|
||||
".gate_proj": (".gate_up_proj", 0),
|
||||
".up_proj": (".gate_up_proj", 1),
|
||||
".q_a_proj": (".fused_qkv_a_proj", 0),
|
||||
".kv_a_proj_with_mqa": (".fused_qkv_a_proj", 1),
|
||||
},
|
||||
)
|
||||
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
|
||||
super().__init__()
|
||||
assert vllm_config.speculative_config is not None
|
||||
self.draft_model_config = vllm_config.speculative_config.draft_model_config
|
||||
self.config = self.draft_model_config.hf_config
|
||||
target_layer_num = vllm_config.model_config.get_num_layers(
|
||||
vllm_config.parallel_config
|
||||
)
|
||||
self.model = K3DSparkModel(
|
||||
vllm_config=vllm_config,
|
||||
start_layer_id=target_layer_num,
|
||||
prefix=maybe_prefix(prefix, "model"),
|
||||
)
|
||||
|
||||
# Assigned by load_dspark_model from the target. Keeping no placeholder
|
||||
# avoids a transient full-vocabulary allocation for this 163k-vocab model.
|
||||
self.lm_head: nn.Module | None = None
|
||||
logit_scale = getattr(self.config, "logit_scale", 1.0)
|
||||
self.logits_processor = LogitsProcessor(
|
||||
self.config.draft_vocab_size, scale=logit_scale
|
||||
)
|
||||
|
||||
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
|
||||
return self.model.embed_input_ids(input_ids)
|
||||
|
||||
def combine_hidden_states(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
||||
return self.model.combine_hidden_states(hidden_states)
|
||||
|
||||
def get_draft_kv_cache_layer_names(self) -> list[str]:
|
||||
return [layer.self_attn.layer_name for layer in self.model.layers]
|
||||
|
||||
def precompute_and_store_context_kv(
|
||||
self,
|
||||
context_states: torch.Tensor,
|
||||
context_positions: torch.Tensor,
|
||||
context_slot_mapping: torch.Tensor | list[torch.Tensor | None] | None = None,
|
||||
) -> None:
|
||||
self.model.precompute_and_store_context_kv(
|
||||
context_states, context_positions, context_slot_mapping
|
||||
)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
inputs_embeds: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
return self.model(input_ids, positions, inputs_embeds)
|
||||
|
||||
def compute_logits(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
||||
assert self.lm_head is not None
|
||||
return self.logits_processor(self.lm_head, hidden_states)
|
||||
|
||||
def compute_draft_logits(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
||||
return self.compute_logits(hidden_states)
|
||||
|
||||
def map_draft_to_target(self, draft_ids: torch.Tensor) -> torch.Tensor:
|
||||
return draft_ids
|
||||
|
||||
def markov_embed(self, token_ids: torch.Tensor) -> torch.Tensor:
|
||||
return self.model.markov_head.embed(token_ids)
|
||||
|
||||
def markov_bias(self, markov_embed: torch.Tensor) -> torch.Tensor:
|
||||
return self.model.markov_head.bias(markov_embed, self.logits_processor)
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
# confidence_head is training-only. The frozen target embedding and LM
|
||||
# head are shared after this draft-specific checkpoint is loaded.
|
||||
loader = AutoWeightsLoader(
|
||||
self,
|
||||
skip_substrs=list(self.checkpoint_skip_substrs),
|
||||
)
|
||||
loaded_weights = loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
|
||||
self.model._build_fused_context_kv_buffers()
|
||||
return loaded_weights
|
||||
@@ -0,0 +1,775 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from collections.abc import Callable
|
||||
|
||||
import torch
|
||||
from einops import rearrange
|
||||
from torch import nn
|
||||
from torch.nn.parameter import Parameter
|
||||
|
||||
from vllm import _custom_ops as ops
|
||||
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed import divide, get_tensor_model_parallel_rank
|
||||
from vllm.forward_context import get_forward_context
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.linear import (
|
||||
ColumnParallelLinear,
|
||||
MergedColumnParallelLinear,
|
||||
RowParallelLinear,
|
||||
)
|
||||
from vllm.model_executor.layers.mamba.gdn.base import GatedDeltaNetAttention
|
||||
from vllm.model_executor.layers.mamba.mamba_utils import (
|
||||
MambaStateDtypeCalculator,
|
||||
MambaStateShapeCalculator,
|
||||
is_conv_state_dim_first,
|
||||
)
|
||||
from vllm.model_executor.layers.mamba.ops.causal_conv1d import (
|
||||
causal_conv1d_fn,
|
||||
causal_conv1d_update,
|
||||
)
|
||||
from vllm.model_executor.layers.mamba.ops.gather_initial_states import (
|
||||
gather_initial_states,
|
||||
)
|
||||
from vllm.model_executor.model_loader.weight_utils import (
|
||||
default_weight_loader,
|
||||
sharded_weight_loader,
|
||||
)
|
||||
from vllm.model_executor.parameter import BasevLLMParameter
|
||||
from vllm.model_executor.utils import set_weight_attrs
|
||||
from vllm.models.kimi_k3.nvidia.kda_metadata import (
|
||||
KimiK3KDAAttentionBackend,
|
||||
KimiK3KDAMetadata,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.third_party.flash_linear_attention.ops.kda import FusedRMSNormGated
|
||||
from vllm.transformers_utils.configs.kimi_linear import KimiLinearConfig
|
||||
from vllm.v1.attention.backend import AttentionBackend
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
_KDA_GATE_LOGBOUND_MIN = -5.0
|
||||
|
||||
|
||||
def a_log_weight_loader(
|
||||
shard_axis: int,
|
||||
) -> Callable[[torch.Tensor, torch.Tensor], None]:
|
||||
"""Load KDA A_log stored as either old 4D or current 1D weights."""
|
||||
|
||||
def loader(param: torch.Tensor, loaded_weight: torch.Tensor) -> None:
|
||||
tp_rank = get_tensor_model_parallel_rank()
|
||||
shard_size = param.data.shape[shard_axis]
|
||||
start_idx = tp_rank * shard_size
|
||||
|
||||
if loaded_weight.dim() == 4:
|
||||
assert loaded_weight.shape[:2] == (1, 1), (
|
||||
f"Expected old A_log shape (1, 1, H, 1), got {loaded_weight.shape}"
|
||||
)
|
||||
assert loaded_weight.shape[-1] == 1, (
|
||||
f"Expected old A_log last dim to be 1, got {loaded_weight.shape}"
|
||||
)
|
||||
loaded_weight = loaded_weight.view(loaded_weight.shape[2])
|
||||
|
||||
loaded_weight = loaded_weight.narrow(shard_axis, start_idx, shard_size)
|
||||
return default_weight_loader(param, loaded_weight)
|
||||
|
||||
return loader
|
||||
|
||||
|
||||
class _KimiGDNMergedColumnParallelLinear(MergedColumnParallelLinear):
|
||||
"""Merged projection with one output replicated across TP ranks."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
input_size: int,
|
||||
output_sizes: list[int],
|
||||
replicated_shard_id: int,
|
||||
tp_size: int,
|
||||
**kwargs,
|
||||
) -> None:
|
||||
self.replicated_shard_id = replicated_shard_id
|
||||
output_sizes = output_sizes.copy()
|
||||
output_sizes[replicated_shard_id] *= tp_size
|
||||
super().__init__(input_size, output_sizes, **kwargs)
|
||||
|
||||
def weight_loader(
|
||||
self,
|
||||
param: Parameter,
|
||||
loaded_weight: torch.Tensor,
|
||||
loaded_shard_id: tuple[int, ...] | int | None = None,
|
||||
) -> None:
|
||||
tp_rank = self.tp_rank
|
||||
param_tp_rank = getattr(param, "tp_rank", None)
|
||||
if loaded_shard_id == self.replicated_shard_id:
|
||||
self.tp_rank = 0
|
||||
if param_tp_rank is not None:
|
||||
param.tp_rank = 0
|
||||
try:
|
||||
super().weight_loader(param, loaded_weight, loaded_shard_id)
|
||||
finally:
|
||||
self.tp_rank = tp_rank
|
||||
if param_tp_rank is not None:
|
||||
param.tp_rank = param_tp_rank
|
||||
|
||||
def weight_loader_v2(
|
||||
self,
|
||||
param: BasevLLMParameter,
|
||||
loaded_weight: torch.Tensor,
|
||||
loaded_shard_id: tuple[int, ...] | int | None = None,
|
||||
) -> None:
|
||||
tp_rank = self.tp_rank
|
||||
param_tp_rank = getattr(param, "tp_rank", None)
|
||||
if loaded_shard_id == self.replicated_shard_id:
|
||||
self.tp_rank = 0
|
||||
if param_tp_rank is not None:
|
||||
param.tp_rank = 0
|
||||
try:
|
||||
super().weight_loader_v2(param, loaded_weight, loaded_shard_id)
|
||||
finally:
|
||||
self.tp_rank = tp_rank
|
||||
if param_tp_rank is not None:
|
||||
param.tp_rank = param_tp_rank
|
||||
|
||||
|
||||
def is_fused_kda_decode_supported(
|
||||
num_heads: int,
|
||||
head_dim: int,
|
||||
conv_width: int,
|
||||
num_spec: int,
|
||||
input_dtype: torch.dtype,
|
||||
conv_state_dtype: torch.dtype,
|
||||
) -> bool:
|
||||
if (
|
||||
num_heads not in (12, 24, 48, 96)
|
||||
or head_dim != 128
|
||||
or conv_width != 4
|
||||
or num_spec != 0
|
||||
or input_dtype != torch.bfloat16
|
||||
or conv_state_dtype != torch.bfloat16
|
||||
or is_conv_state_dim_first()
|
||||
or not hasattr(torch.ops._C, "fused_kda_decode")
|
||||
):
|
||||
return False
|
||||
# SM90 is architecture-specific; SM10x and SM12x use family binaries.
|
||||
return (
|
||||
current_platform.is_device_capability(90)
|
||||
or current_platform.is_device_capability_family(100)
|
||||
or current_platform.is_device_capability_family(120)
|
||||
)
|
||||
|
||||
|
||||
def is_flashkda_supported(
|
||||
head_dim: int,
|
||||
dtype: torch.dtype,
|
||||
lower_bound: float | None,
|
||||
) -> bool:
|
||||
capability = current_platform.get_device_capability()
|
||||
return (
|
||||
capability is not None
|
||||
and capability.major in (9, 10, 12)
|
||||
and head_dim == 128
|
||||
and dtype == torch.bfloat16
|
||||
and lower_bound is not None
|
||||
)
|
||||
|
||||
|
||||
def _flashkda_prefill(
|
||||
q: torch.Tensor,
|
||||
k: torch.Tensor,
|
||||
v: torch.Tensor,
|
||||
g: torch.Tensor,
|
||||
beta: torch.Tensor,
|
||||
A_log: torch.Tensor,
|
||||
dt_bias: torch.Tensor,
|
||||
lower_bound: float,
|
||||
initial_state: torch.Tensor,
|
||||
cu_seqlens: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
import vllm._flashkda_C # noqa: F401
|
||||
|
||||
out = torch.empty(v.shape, dtype=v.dtype, device=v.device)
|
||||
final_state = torch.empty_like(initial_state)
|
||||
workspace = torch.empty(
|
||||
torch.ops._flashkda_C.get_workspace_size(
|
||||
q.shape[0] * q.shape[1],
|
||||
q.shape[2],
|
||||
cu_seqlens.numel() - 1,
|
||||
),
|
||||
dtype=torch.uint8,
|
||||
device=q.device,
|
||||
)
|
||||
# FlashKDA hardcodes dense Q/K/V/G strides. Beta may be row-strided because
|
||||
# FlashKDA materializes its transposed [H, T] layout internally.
|
||||
# TODO: Teach FlashKDA to consume beta in [T, H] layout directly instead
|
||||
# of transposing it to contiguous [H, T] storage internally.
|
||||
torch.ops._flashkda_C.fwd(
|
||||
q.contiguous(),
|
||||
k.contiguous(),
|
||||
v.contiguous(),
|
||||
g.contiguous(),
|
||||
beta,
|
||||
q.shape[-1] ** -0.5,
|
||||
out,
|
||||
workspace,
|
||||
A_log.contiguous(),
|
||||
dt_bias.view(-1, q.shape[-1]).contiguous(),
|
||||
lower_bound,
|
||||
initial_state.contiguous(),
|
||||
final_state,
|
||||
cu_seqlens.contiguous(),
|
||||
)
|
||||
return out, final_state
|
||||
|
||||
|
||||
def resolve_kda_prefill_backend(
|
||||
backend: str,
|
||||
head_dim: int,
|
||||
dtype: torch.dtype,
|
||||
lower_bound: float | None,
|
||||
) -> str:
|
||||
if backend not in ("auto", "triton", "flashkda"):
|
||||
raise ValueError(f"Unsupported KDA prefill backend: {backend}")
|
||||
supported = is_flashkda_supported(head_dim, dtype, lower_bound)
|
||||
if backend == "flashkda" and not supported:
|
||||
raise RuntimeError(
|
||||
"FlashKDA requires CUDA SM90/SM10x/SM12x, bfloat16, "
|
||||
"head_dim=128, and a bounded KDA gate."
|
||||
)
|
||||
if supported and backend != "triton":
|
||||
logger.info_once("Using FlashKDA KDA prefill backend.")
|
||||
return "flashkda"
|
||||
return "triton"
|
||||
|
||||
|
||||
def _make_decode_conv1d_weight_loader(
|
||||
dims: list[int],
|
||||
tp_size: int,
|
||||
tp_rank: int,
|
||||
decode_conv1d_weight: torch.Tensor | None,
|
||||
) -> Callable[..., None]:
|
||||
sharded_dims = [dim // tp_size for dim in dims]
|
||||
|
||||
def weight_loader(
|
||||
param: torch.Tensor,
|
||||
loaded_weight: torch.Tensor,
|
||||
loaded_shard_id: int,
|
||||
) -> None:
|
||||
if loaded_weight.dim() == 2:
|
||||
loaded_weight = loaded_weight.unsqueeze(1)
|
||||
shard_size = sharded_dims[loaded_shard_id]
|
||||
source_start = tp_rank * shard_size
|
||||
target_start = sum(sharded_dims[:loaded_shard_id])
|
||||
loaded_shard = loaded_weight[source_start : source_start + shard_size]
|
||||
param.data[target_start : target_start + shard_size].copy_(loaded_shard)
|
||||
if decode_conv1d_weight is not None and not param.is_meta:
|
||||
decode_conv1d_weight[loaded_shard_id].copy_(
|
||||
loaded_shard.squeeze(1).transpose(0, 1)
|
||||
)
|
||||
|
||||
return weight_loader
|
||||
|
||||
|
||||
def _make_decode_norm_weight_loader(
|
||||
decode_norm_weight: torch.Tensor,
|
||||
) -> Callable[..., None]:
|
||||
def weight_loader(param: torch.Tensor, loaded_weight: torch.Tensor) -> None:
|
||||
default_weight_loader(param, loaded_weight)
|
||||
if not param.is_meta:
|
||||
decode_norm_weight.copy_(param.data)
|
||||
|
||||
return weight_loader
|
||||
|
||||
|
||||
class KimiK3DeltaAttention(GatedDeltaNetAttention):
|
||||
def get_attn_backend(self) -> type[AttentionBackend]:
|
||||
return KimiK3KDAAttentionBackend
|
||||
|
||||
def get_state_dtype(
|
||||
self,
|
||||
) -> tuple[torch.dtype, torch.dtype]:
|
||||
if self.model_config is None or self.cache_config is None:
|
||||
raise ValueError("model_config and cache_config must be set")
|
||||
return MambaStateDtypeCalculator.kda_state_dtype(
|
||||
self.model_config.dtype, self.cache_config.mamba_cache_dtype
|
||||
)
|
||||
|
||||
def get_state_shape(
|
||||
self,
|
||||
) -> tuple[tuple[int, ...], tuple[int, ...]]:
|
||||
return MambaStateShapeCalculator.kda_state_shape(
|
||||
self.tp_size,
|
||||
self.num_heads,
|
||||
self.head_dim,
|
||||
conv_kernel_size=self.conv_size,
|
||||
num_spec=self.num_spec,
|
||||
)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: KimiLinearConfig,
|
||||
vllm_config: VllmConfig,
|
||||
prefix: str = "",
|
||||
) -> None:
|
||||
super().__init__(config, vllm_config, prefix)
|
||||
|
||||
kda_config = config.linear_attn_config # type: ignore[attr-defined]
|
||||
assert kda_config is not None, "linear_attn_config must be set"
|
||||
self.head_dim = kda_config["head_dim"]
|
||||
self.num_heads = kda_config["num_heads"]
|
||||
assert self.num_heads % self.tp_size == 0
|
||||
self.local_num_heads = divide(self.num_heads, self.tp_size)
|
||||
self.projection_size = self.head_dim * self.num_heads
|
||||
self.local_projection_size = divide(self.projection_size, self.tp_size)
|
||||
self.conv_size = kda_config["short_conv_kernel_size"]
|
||||
assert kda_config.get("use_full_rank_gate", False), (
|
||||
"KimiK3DeltaAttention requires a full-rank gate"
|
||||
)
|
||||
|
||||
# Keep f_a before the narrow beta shard, then pad each TP-local row
|
||||
# to select the aligned BF16 GEMM path.
|
||||
qkvg_output_sizes = [self.projection_size] * 4
|
||||
in_proj_output_sizes = qkvg_output_sizes + [
|
||||
self.head_dim,
|
||||
self.num_heads,
|
||||
]
|
||||
local_output_size = (
|
||||
4 * self.local_projection_size + self.head_dim + self.local_num_heads
|
||||
)
|
||||
self.in_proj_padding = -local_output_size % 16
|
||||
if self.in_proj_padding:
|
||||
in_proj_output_sizes.append(self.in_proj_padding * self.tp_size)
|
||||
self.in_proj_qkvgfab = _KimiGDNMergedColumnParallelLinear(
|
||||
self.hidden_size,
|
||||
in_proj_output_sizes,
|
||||
replicated_shard_id=4,
|
||||
tp_size=self.tp_size,
|
||||
bias=False,
|
||||
quant_config=self.quant_config,
|
||||
prefix=f"{prefix}.in_proj_qkvgfab",
|
||||
)
|
||||
if self.in_proj_padding:
|
||||
self.in_proj_qkvgfab.weight.data[-self.in_proj_padding :].zero_()
|
||||
|
||||
self.f_b_proj = ColumnParallelLinear(
|
||||
self.head_dim,
|
||||
self.projection_size,
|
||||
bias=False,
|
||||
quant_config=self.quant_config,
|
||||
prefix=f"{prefix}.f_b_proj",
|
||||
)
|
||||
self.dt_bias = nn.Parameter(
|
||||
torch.empty(self.local_projection_size, dtype=torch.float32)
|
||||
)
|
||||
set_weight_attrs(self.dt_bias, {"weight_loader": sharded_weight_loader(0)})
|
||||
|
||||
# One packed parameter and cache let decode run a single conv update.
|
||||
# Prefill slices them back into Q/K/V to obtain dense outputs cheaply.
|
||||
self.conv1d = ColumnParallelLinear(
|
||||
input_size=self.conv_size,
|
||||
output_size=3 * self.projection_size,
|
||||
bias=False,
|
||||
params_dtype=torch.float32,
|
||||
prefix=f"{prefix}.conv1d",
|
||||
)
|
||||
self.conv1d.weight.data = self.conv1d.weight.data.unsqueeze(1)
|
||||
# Keep a width-major copy for fused decode without changing the layout
|
||||
# consumed by the prefill and fallback decode kernels.
|
||||
conv_state_dtype, _ = self.get_state_dtype()
|
||||
decode_conv1d_weight = None
|
||||
if is_fused_kda_decode_supported(
|
||||
self.local_num_heads,
|
||||
self.head_dim,
|
||||
self.conv_size,
|
||||
self.num_spec,
|
||||
vllm_config.model_config.dtype,
|
||||
conv_state_dtype,
|
||||
):
|
||||
logger.info_once("Fused KDA decode kernel (conv+KDA+norm) is enabled.")
|
||||
decode_conv1d_weight = torch.empty(
|
||||
3,
|
||||
self.conv_size,
|
||||
self.local_projection_size,
|
||||
dtype=self.conv1d.weight.dtype,
|
||||
device=self.conv1d.weight.device,
|
||||
)
|
||||
self.register_buffer(
|
||||
"decode_conv1d_weight", decode_conv1d_weight, persistent=False
|
||||
)
|
||||
delattr(self.conv1d.weight, "weight_loader")
|
||||
set_weight_attrs(
|
||||
self.conv1d.weight,
|
||||
{
|
||||
"weight_loader": _make_decode_conv1d_weight_loader(
|
||||
[self.projection_size] * 3,
|
||||
self.tp_size,
|
||||
self.tp_rank,
|
||||
decode_conv1d_weight,
|
||||
)
|
||||
},
|
||||
)
|
||||
|
||||
self.A_log = nn.Parameter(
|
||||
torch.empty(self.local_num_heads, dtype=torch.float32)
|
||||
)
|
||||
set_weight_attrs(self.A_log, {"weight_loader": a_log_weight_loader(0)})
|
||||
|
||||
self.gate_lower_bound: float | None = kda_config.get("gate_lower_bound", None)
|
||||
if self.gate_lower_bound is not None:
|
||||
assert _KDA_GATE_LOGBOUND_MIN <= self.gate_lower_bound < 0, (
|
||||
"KDA gate lower bound must be in "
|
||||
f"[{_KDA_GATE_LOGBOUND_MIN}, 0). "
|
||||
f"Got {self.gate_lower_bound}."
|
||||
)
|
||||
|
||||
additional_config = vllm_config.additional_config
|
||||
backend = (
|
||||
additional_config.get("kda_prefill_backend", "auto")
|
||||
if isinstance(additional_config, dict)
|
||||
else "auto"
|
||||
)
|
||||
self.kda_prefill_backend = resolve_kda_prefill_backend(
|
||||
backend,
|
||||
self.head_dim,
|
||||
vllm_config.model_config.dtype,
|
||||
self.gate_lower_bound,
|
||||
)
|
||||
|
||||
self.o_norm = FusedRMSNormGated(self.head_dim, activation="sigmoid")
|
||||
decode_norm_weight = None
|
||||
if decode_conv1d_weight is not None:
|
||||
decode_norm_weight = torch.empty(
|
||||
self.head_dim,
|
||||
dtype=torch.float32,
|
||||
device=self.o_norm.weight.device,
|
||||
)
|
||||
self.register_buffer("decode_norm_weight", decode_norm_weight, persistent=False)
|
||||
if decode_norm_weight is not None:
|
||||
# Upcast once while loading; direct BF16 norm weights slow the
|
||||
# fully fused decode kernel.
|
||||
if hasattr(self.o_norm.weight, "weight_loader"):
|
||||
delattr(self.o_norm.weight, "weight_loader")
|
||||
set_weight_attrs(
|
||||
self.o_norm.weight,
|
||||
{"weight_loader": _make_decode_norm_weight_loader(decode_norm_weight)},
|
||||
)
|
||||
self.o_proj = RowParallelLinear(
|
||||
self.projection_size,
|
||||
self.hidden_size,
|
||||
bias=False,
|
||||
quant_config=self.quant_config,
|
||||
prefix=f"{prefix}.o_proj",
|
||||
)
|
||||
|
||||
compilation_config = vllm_config.compilation_config
|
||||
if prefix in compilation_config.static_forward_context:
|
||||
raise ValueError(f"Duplicate layer name: {prefix}")
|
||||
compilation_config.static_forward_context[prefix] = self
|
||||
|
||||
def forward(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
num_tokens = hidden_states.size(0)
|
||||
projected_qkvgfab = self.in_proj_qkvgfab(hidden_states)[0]
|
||||
split_sizes = [
|
||||
3 * self.local_projection_size,
|
||||
self.local_projection_size,
|
||||
self.head_dim,
|
||||
self.local_num_heads,
|
||||
]
|
||||
if self.in_proj_padding:
|
||||
split_sizes.append(self.in_proj_padding)
|
||||
projected = projected_qkvgfab.split(split_sizes, dim=-1)
|
||||
mixed_qkv, g_proj_states, f_a, beta = projected[:4]
|
||||
|
||||
g1 = self.f_b_proj(f_a)[0]
|
||||
beta = beta.unsqueeze(0)
|
||||
g1 = rearrange(g1, "n (h d) -> 1 n h d", d=self.head_dim)
|
||||
g2 = rearrange(g_proj_states, "... (h d) -> ... h d", d=self.head_dim)
|
||||
core_attn_out = torch.empty(
|
||||
(1, num_tokens, self.local_num_heads, self.head_dim),
|
||||
dtype=hidden_states.dtype,
|
||||
device=hidden_states.device,
|
||||
)
|
||||
self._forward(
|
||||
mixed_qkv=mixed_qkv,
|
||||
g1=g1,
|
||||
g2=g2,
|
||||
beta=beta,
|
||||
core_attn_out=core_attn_out,
|
||||
)
|
||||
core_attn_out = rearrange(core_attn_out, "1 n h d -> n (h d)")
|
||||
return self.o_proj(core_attn_out)[0]
|
||||
|
||||
@eager_break_during_capture
|
||||
def _forward(
|
||||
self,
|
||||
mixed_qkv: torch.Tensor,
|
||||
g1: torch.Tensor,
|
||||
g2: torch.Tensor,
|
||||
beta: torch.Tensor,
|
||||
core_attn_out: torch.Tensor,
|
||||
) -> None:
|
||||
forward_context = get_forward_context()
|
||||
attn_metadata_raw = forward_context.attn_metadata
|
||||
if attn_metadata_raw is None:
|
||||
return
|
||||
|
||||
from vllm.models.kimi_k3.nvidia.ops.third_party.kda import (
|
||||
chunk_kda_with_fused_gate,
|
||||
fused_recurrent_kda,
|
||||
fused_recurrent_kda_packed_decode,
|
||||
)
|
||||
|
||||
assert isinstance(attn_metadata_raw, dict)
|
||||
attn_metadata_narrowed = attn_metadata_raw[self.prefix]
|
||||
assert isinstance(attn_metadata_narrowed, KimiK3KDAMetadata)
|
||||
m = attn_metadata_narrowed
|
||||
has_initial_state = m.has_initial_state
|
||||
non_spec_query_start_loc = m.non_spec_query_start_loc
|
||||
non_spec_state_indices_tensor = m.non_spec_state_indices_tensor
|
||||
spec_token_indx = m.spec_token_indx
|
||||
non_spec_token_indx = m.non_spec_token_indx
|
||||
spec_state_indices_tensor = m.spec_state_indices_tensor
|
||||
spec_query_start_loc = m.spec_query_start_loc
|
||||
num_accepted_tokens = m.num_accepted_tokens
|
||||
num_actual_tokens = m.num_actual_tokens
|
||||
has_spec_decode = m.num_spec_decodes > 0
|
||||
mixed_qkv = mixed_qkv[:num_actual_tokens]
|
||||
g1 = g1[:, :num_actual_tokens]
|
||||
beta = beta[:, :num_actual_tokens]
|
||||
|
||||
conv_state, recurrent_state = self.kv_cache
|
||||
# The convolution kernels consume (..., dim, width - 1).
|
||||
if not is_conv_state_dim_first():
|
||||
conv_state = conv_state.transpose(-1, -2)
|
||||
|
||||
if (
|
||||
self.decode_conv1d_weight is not None
|
||||
and self.decode_norm_weight is not None
|
||||
and not has_spec_decode
|
||||
and m.num_prefills == 0
|
||||
and m.num_decodes > 0
|
||||
):
|
||||
assert non_spec_state_indices_tensor is not None
|
||||
ops.fused_kda_decode(
|
||||
x=mixed_qkv,
|
||||
weight=self.decode_conv1d_weight,
|
||||
bias=self.conv1d.bias,
|
||||
conv_state=conv_state,
|
||||
raw_g=g1,
|
||||
raw_beta=beta,
|
||||
A_log=self.A_log,
|
||||
dt_bias=self.dt_bias,
|
||||
state_indices=non_spec_state_indices_tensor[:num_actual_tokens],
|
||||
state=recurrent_state,
|
||||
out=core_attn_out[:, :num_actual_tokens],
|
||||
lower_bound=self.gate_lower_bound,
|
||||
output_gate=g2[:num_actual_tokens],
|
||||
norm_weight=self.decode_norm_weight,
|
||||
norm_eps=self.o_norm.eps,
|
||||
)
|
||||
return
|
||||
|
||||
conv_weights = self.conv1d.weight.view(
|
||||
self.conv1d.weight.size(0), self.conv1d.weight.size(2)
|
||||
)
|
||||
q_conv_weight, k_conv_weight, v_conv_weight = conv_weights.split(
|
||||
self.local_projection_size, dim=0
|
||||
)
|
||||
q_conv_state, k_conv_state, v_conv_state = conv_state.split(
|
||||
self.local_projection_size, dim=-2
|
||||
)
|
||||
|
||||
# Separate multi-query speculative tokens from prefill/plain decode.
|
||||
if has_spec_decode:
|
||||
if m.num_prefills == 0 and m.num_decodes == 0:
|
||||
mixed_qkv_spec = mixed_qkv
|
||||
g1_spec, beta_spec = g1, beta
|
||||
mixed_qkv_ns = g1_ns = beta_ns = None
|
||||
else:
|
||||
assert spec_token_indx is not None
|
||||
assert non_spec_token_indx is not None
|
||||
mixed_qkv_spec = mixed_qkv.index_select(0, spec_token_indx)
|
||||
g1_spec = g1.index_select(1, spec_token_indx)
|
||||
beta_spec = beta.index_select(1, spec_token_indx)
|
||||
mixed_qkv_ns = mixed_qkv.index_select(0, non_spec_token_indx)
|
||||
g1_ns = g1.index_select(1, non_spec_token_indx)
|
||||
beta_ns = beta.index_select(1, non_spec_token_indx)
|
||||
else:
|
||||
mixed_qkv_spec = g1_spec = beta_spec = None
|
||||
mixed_qkv_ns, g1_ns, beta_ns = mixed_qkv, g1, beta
|
||||
|
||||
# Spec-decode multi-query path.
|
||||
core_attn_out_spec = None
|
||||
if has_spec_decode:
|
||||
assert spec_state_indices_tensor is not None
|
||||
assert spec_query_start_loc is not None
|
||||
spec_conv_indices = spec_state_indices_tensor[:, 0][: m.num_spec_decodes]
|
||||
spec_max_query_len = spec_state_indices_tensor.size(-1)
|
||||
spec_conv_out = torch.empty_like(mixed_qkv_spec)
|
||||
mixed_qkv_spec = causal_conv1d_update(
|
||||
mixed_qkv_spec,
|
||||
conv_state,
|
||||
conv_weights,
|
||||
self.conv1d.bias,
|
||||
activation="silu",
|
||||
conv_state_indices=spec_conv_indices,
|
||||
num_accepted_tokens=num_accepted_tokens,
|
||||
query_start_loc=spec_query_start_loc,
|
||||
max_query_len=spec_max_query_len,
|
||||
validate_data=False,
|
||||
out=spec_conv_out,
|
||||
)
|
||||
q_spec, k_spec, v_spec = (
|
||||
rearrange(x, "n (h d) -> 1 n h d", d=self.head_dim)
|
||||
for x in mixed_qkv_spec.split(self.local_projection_size, dim=-1)
|
||||
)
|
||||
spec_cu_seqlens = spec_query_start_loc[: m.num_spec_decodes + 1]
|
||||
spec_out = (
|
||||
core_attn_out[:, : q_spec.shape[1]]
|
||||
if m.num_prefills == 0 and m.num_decodes == 0
|
||||
else None
|
||||
)
|
||||
core_attn_out_spec, _ = fused_recurrent_kda(
|
||||
q=q_spec,
|
||||
k=k_spec,
|
||||
v=v_spec,
|
||||
raw_g=g1_spec,
|
||||
raw_beta=beta_spec,
|
||||
A_log=self.A_log,
|
||||
dt_bias=self.dt_bias,
|
||||
lower_bound=self.gate_lower_bound,
|
||||
initial_state=recurrent_state,
|
||||
cu_seqlens=spec_cu_seqlens,
|
||||
ssm_state_indices=spec_state_indices_tensor,
|
||||
num_accepted_tokens=num_accepted_tokens,
|
||||
out=spec_out,
|
||||
)
|
||||
|
||||
# Prefill or plain-decode path.
|
||||
core_attn_out_non_spec = None
|
||||
if mixed_qkv_ns is not None:
|
||||
assert g1_ns is not None and beta_ns is not None
|
||||
if m.num_prefills > 0:
|
||||
q_ns, k_ns, v_ns = mixed_qkv_ns.split(
|
||||
self.local_projection_size, dim=-1
|
||||
)
|
||||
|
||||
# Separate convolution calls accept row-strided packed inputs
|
||||
# and produce dense Q/K/V without an additional V copy.
|
||||
def _prefill_conv(
|
||||
x: torch.Tensor,
|
||||
state: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
return causal_conv1d_fn(
|
||||
x.transpose(0, 1),
|
||||
weight,
|
||||
None,
|
||||
activation="silu",
|
||||
conv_states=state,
|
||||
has_initial_state=has_initial_state,
|
||||
cache_indices=non_spec_state_indices_tensor,
|
||||
query_start_loc=non_spec_query_start_loc,
|
||||
metadata=m,
|
||||
).transpose(0, 1)
|
||||
|
||||
q_ns = _prefill_conv(q_ns, q_conv_state, q_conv_weight)
|
||||
k_ns = _prefill_conv(k_ns, k_conv_state, k_conv_weight)
|
||||
v_ns = _prefill_conv(v_ns, v_conv_state, v_conv_weight)
|
||||
q_ns, k_ns, v_ns = (
|
||||
rearrange(x, "n (h d) -> 1 n h d", d=self.head_dim)
|
||||
for x in (q_ns, k_ns, v_ns)
|
||||
)
|
||||
|
||||
assert non_spec_state_indices_tensor is not None
|
||||
assert has_initial_state is not None
|
||||
initial_state = gather_initial_states(
|
||||
recurrent_state,
|
||||
non_spec_state_indices_tensor,
|
||||
has_initial_state,
|
||||
)
|
||||
if self.kda_prefill_backend == "flashkda":
|
||||
assert self.gate_lower_bound is not None
|
||||
(
|
||||
core_attn_out_non_spec,
|
||||
last_recurrent_state,
|
||||
) = _flashkda_prefill(
|
||||
q=q_ns,
|
||||
k=k_ns,
|
||||
v=v_ns,
|
||||
g=g1_ns,
|
||||
beta=beta_ns,
|
||||
A_log=self.A_log,
|
||||
dt_bias=self.dt_bias,
|
||||
lower_bound=self.gate_lower_bound,
|
||||
initial_state=initial_state,
|
||||
cu_seqlens=non_spec_query_start_loc,
|
||||
)
|
||||
else:
|
||||
(
|
||||
core_attn_out_non_spec,
|
||||
last_recurrent_state,
|
||||
) = chunk_kda_with_fused_gate(
|
||||
q=q_ns,
|
||||
k=k_ns,
|
||||
v=v_ns,
|
||||
raw_g=g1_ns,
|
||||
raw_beta=beta_ns,
|
||||
A_log=self.A_log,
|
||||
g_bias=self.dt_bias,
|
||||
lower_bound=self.gate_lower_bound,
|
||||
initial_state=initial_state,
|
||||
output_final_state=True,
|
||||
use_qk_l2norm_in_kernel=True,
|
||||
cu_seqlens=non_spec_query_start_loc,
|
||||
)
|
||||
recurrent_state[non_spec_state_indices_tensor] = last_recurrent_state
|
||||
else:
|
||||
# Pure non-speculative decode.
|
||||
assert non_spec_state_indices_tensor is not None
|
||||
decode_conv_indices = non_spec_state_indices_tensor[
|
||||
: mixed_qkv_ns.size(0)
|
||||
]
|
||||
packed_conv_out = torch.empty_like(mixed_qkv_ns)
|
||||
mixed_qkv_ns = causal_conv1d_update(
|
||||
mixed_qkv_ns,
|
||||
conv_state,
|
||||
conv_weights,
|
||||
self.conv1d.bias,
|
||||
activation="silu",
|
||||
conv_state_indices=decode_conv_indices,
|
||||
validate_data=True,
|
||||
out=packed_conv_out,
|
||||
)
|
||||
(
|
||||
core_attn_out_non_spec,
|
||||
_,
|
||||
) = fused_recurrent_kda_packed_decode(
|
||||
mixed_qkv=mixed_qkv_ns,
|
||||
raw_g=g1_ns,
|
||||
raw_beta=beta_ns,
|
||||
A_log=self.A_log,
|
||||
dt_bias=self.dt_bias,
|
||||
lower_bound=self.gate_lower_bound,
|
||||
initial_state=recurrent_state,
|
||||
state_indices=decode_conv_indices,
|
||||
)
|
||||
|
||||
# Restore the scheduler's original token order for mixed batches.
|
||||
if core_attn_out_spec is not None and core_attn_out_non_spec is not None:
|
||||
core_attn_out.index_copy_(1, spec_token_indx, core_attn_out_spec)
|
||||
core_attn_out.index_copy_(1, non_spec_token_indx, core_attn_out_non_spec)
|
||||
elif core_attn_out_non_spec is not None:
|
||||
# TODO: prefill and decode kernels write directly to core_attn_out
|
||||
core_attn_out[0, :num_actual_tokens] = core_attn_out_non_spec[
|
||||
0, :num_actual_tokens
|
||||
]
|
||||
else:
|
||||
assert core_attn_out_spec is not None
|
||||
# Triton normalizes in place, so this is a self-copy with no device
|
||||
# work. Keep it for the out-of-place native implementation.
|
||||
core_attn_out.copy_(self.o_norm(core_attn_out, g2))
|
||||
@@ -0,0 +1,496 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Kimi-K3 specialization of GDN attention metadata.
|
||||
|
||||
The request classification and cudagraph staging intentionally mirror
|
||||
``GDNAttentionMetadataBuilder``. Kimi-K3 builds the metadata required by its
|
||||
prefill KDA kernel internally, so this builder omits the shared FLA chunk
|
||||
metadata construction.
|
||||
"""
|
||||
|
||||
from dataclasses import dataclass
|
||||
from functools import cache
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.utils.torch_utils import async_tensor_h2d
|
||||
from vllm.v1.attention.backend import CommonAttentionMetadata
|
||||
from vllm.v1.attention.backends.gdn_attn import (
|
||||
GDNAttentionBackend,
|
||||
GDNAttentionMetadata,
|
||||
GDNAttentionMetadataBuilder,
|
||||
)
|
||||
from vllm.v1.attention.backends.utils import (
|
||||
NULL_BLOCK_ID,
|
||||
compute_causal_conv1d_metadata,
|
||||
split_decodes_and_prefills,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import MambaSpec
|
||||
|
||||
|
||||
@cache
|
||||
def _metadata_launch_pdl() -> bool:
|
||||
return current_platform.is_arch_support_pdl()
|
||||
|
||||
|
||||
@triton.jit(do_not_specialize=["num_requests"])
|
||||
def _get_aligned_state_indices_kernel(
|
||||
block_table_ptr,
|
||||
seq_lens_ptr,
|
||||
state_indices_ptr,
|
||||
block_table_stride_0: tl.constexpr,
|
||||
block_table_stride_1: tl.constexpr,
|
||||
seq_lens_stride: tl.constexpr,
|
||||
state_indices_stride_0: tl.constexpr,
|
||||
state_indices_stride_1: tl.constexpr,
|
||||
num_requests,
|
||||
CACHE_BLOCK_SIZE: tl.constexpr,
|
||||
NUM_STATE_SLOTS: tl.constexpr,
|
||||
BLOCK_STATE_SLOTS: tl.constexpr,
|
||||
BLOCK_ROWS: tl.constexpr,
|
||||
launch_pdl: tl.constexpr,
|
||||
):
|
||||
if launch_pdl:
|
||||
tl.extra.cuda.gdc_wait()
|
||||
tl.extra.cuda.gdc_launch_dependents()
|
||||
|
||||
rows = tl.program_id(0) * BLOCK_ROWS + tl.arange(0, BLOCK_ROWS)
|
||||
valid_row = rows < num_requests
|
||||
seq_lens = tl.load(
|
||||
seq_lens_ptr + rows * seq_lens_stride,
|
||||
mask=valid_row,
|
||||
other=1,
|
||||
)
|
||||
# Triton truncates signed division toward zero, unlike PyTorch floor
|
||||
# division. Clamping makes both semantics equivalent for seq_lens <= 0.
|
||||
first_state_slot = tl.maximum((seq_lens - 1) // CACHE_BLOCK_SIZE, 0)
|
||||
|
||||
state_slots = tl.arange(0, BLOCK_STATE_SLOTS)
|
||||
valid_state_slot = state_slots < NUM_STATE_SLOTS
|
||||
state_indices = tl.load(
|
||||
block_table_ptr
|
||||
+ rows[:, None] * block_table_stride_0
|
||||
+ (first_state_slot[:, None] + state_slots[None, :]) * block_table_stride_1,
|
||||
mask=valid_row[:, None] & valid_state_slot[None, :],
|
||||
)
|
||||
tl.store(
|
||||
state_indices_ptr
|
||||
+ rows[:, None] * state_indices_stride_0
|
||||
+ state_slots[None, :] * state_indices_stride_1,
|
||||
state_indices,
|
||||
mask=valid_row[:, None] & valid_state_slot[None, :],
|
||||
)
|
||||
|
||||
|
||||
def _mamba_get_block_table_tensor(
|
||||
block_table: torch.Tensor,
|
||||
seq_lens: torch.Tensor,
|
||||
kv_cache_spec: MambaSpec,
|
||||
mamba_cache_mode: str,
|
||||
) -> torch.Tensor:
|
||||
if mamba_cache_mode in ("all", "none"):
|
||||
return block_table
|
||||
|
||||
assert block_table.is_cuda and seq_lens.is_cuda
|
||||
num_requests = block_table.shape[0]
|
||||
num_state_slots = 1 + kv_cache_spec.num_speculative_blocks
|
||||
state_indices = torch.empty(
|
||||
(num_requests, num_state_slots),
|
||||
dtype=block_table.dtype,
|
||||
device=block_table.device,
|
||||
)
|
||||
BLOCK_ROWS = 32
|
||||
grid = (triton.cdiv(num_requests, BLOCK_ROWS),)
|
||||
_get_aligned_state_indices_kernel[grid](
|
||||
block_table,
|
||||
seq_lens,
|
||||
state_indices,
|
||||
block_table.stride(0),
|
||||
block_table.stride(1),
|
||||
seq_lens.stride(0),
|
||||
state_indices.stride(0),
|
||||
state_indices.stride(1),
|
||||
num_requests,
|
||||
CACHE_BLOCK_SIZE=kv_cache_spec.block_size,
|
||||
NUM_STATE_SLOTS=num_state_slots,
|
||||
BLOCK_STATE_SLOTS=triton.next_power_of_2(num_state_slots),
|
||||
BLOCK_ROWS=BLOCK_ROWS,
|
||||
num_warps=1,
|
||||
launch_pdl=_metadata_launch_pdl(),
|
||||
)
|
||||
return state_indices
|
||||
|
||||
|
||||
@triton.jit(do_not_specialize=["num_spec_decodes", "batch_size"])
|
||||
def _stage_spec_decode_metadata_kernel(
|
||||
state_indices_ptr,
|
||||
query_start_loc_ptr,
|
||||
num_accepted_tokens_ptr,
|
||||
staged_state_indices_ptr,
|
||||
staged_query_start_loc_ptr,
|
||||
staged_num_accepted_tokens_ptr,
|
||||
state_indices_stride_0: tl.constexpr,
|
||||
state_indices_stride_1: tl.constexpr,
|
||||
staged_state_indices_stride_0: tl.constexpr,
|
||||
staged_state_indices_stride_1: tl.constexpr,
|
||||
num_spec_decodes,
|
||||
batch_size,
|
||||
NUM_STATE_SLOTS: tl.constexpr,
|
||||
BLOCK_STATE_SLOTS: tl.constexpr,
|
||||
NULL_STATE_ID: tl.constexpr,
|
||||
BLOCK_ROWS: tl.constexpr,
|
||||
launch_pdl: tl.constexpr,
|
||||
):
|
||||
if launch_pdl:
|
||||
tl.extra.cuda.gdc_wait()
|
||||
tl.extra.cuda.gdc_launch_dependents()
|
||||
|
||||
rows = tl.program_id(0) * BLOCK_ROWS + tl.arange(0, BLOCK_ROWS)
|
||||
real_request = rows < num_spec_decodes
|
||||
|
||||
state_slots = tl.arange(0, BLOCK_STATE_SLOTS)
|
||||
valid_state_slot = state_slots < NUM_STATE_SLOTS
|
||||
state_indices = tl.load(
|
||||
state_indices_ptr
|
||||
+ rows[:, None] * state_indices_stride_0
|
||||
+ state_slots[None, :] * state_indices_stride_1,
|
||||
mask=real_request[:, None] & valid_state_slot[None, :],
|
||||
other=NULL_STATE_ID,
|
||||
)
|
||||
tl.store(
|
||||
staged_state_indices_ptr
|
||||
+ rows[:, None] * staged_state_indices_stride_0
|
||||
+ state_slots[None, :] * staged_state_indices_stride_1,
|
||||
state_indices,
|
||||
mask=(rows < batch_size)[:, None] & valid_state_slot[None, :],
|
||||
)
|
||||
|
||||
query_row = tl.minimum(rows, num_spec_decodes)
|
||||
query_start_loc = tl.load(
|
||||
query_start_loc_ptr + query_row,
|
||||
mask=rows <= batch_size,
|
||||
)
|
||||
tl.store(
|
||||
staged_query_start_loc_ptr + rows,
|
||||
query_start_loc,
|
||||
mask=rows <= batch_size,
|
||||
)
|
||||
|
||||
num_accepted_tokens = tl.load(
|
||||
num_accepted_tokens_ptr + rows,
|
||||
mask=real_request,
|
||||
other=1,
|
||||
)
|
||||
tl.store(
|
||||
staged_num_accepted_tokens_ptr + rows,
|
||||
num_accepted_tokens,
|
||||
mask=rows < batch_size,
|
||||
)
|
||||
|
||||
|
||||
def stage_spec_decode_metadata(
|
||||
state_indices: torch.Tensor,
|
||||
query_start_loc: torch.Tensor,
|
||||
num_accepted_tokens: torch.Tensor,
|
||||
staged_state_indices: torch.Tensor,
|
||||
staged_query_start_loc: torch.Tensor,
|
||||
staged_num_accepted_tokens: torch.Tensor,
|
||||
*,
|
||||
num_spec_decodes: int,
|
||||
) -> None:
|
||||
"""Stage speculative-decode metadata into CUDA-graph buffers."""
|
||||
assert state_indices.is_cuda
|
||||
assert state_indices.ndim == 2
|
||||
batch_size, num_state_slots = staged_state_indices.shape
|
||||
BLOCK_ROWS = 32
|
||||
grid = (triton.cdiv(batch_size + 1, BLOCK_ROWS),)
|
||||
_stage_spec_decode_metadata_kernel[grid](
|
||||
state_indices,
|
||||
query_start_loc,
|
||||
num_accepted_tokens,
|
||||
staged_state_indices,
|
||||
staged_query_start_loc,
|
||||
staged_num_accepted_tokens,
|
||||
state_indices.stride(0),
|
||||
state_indices.stride(1),
|
||||
staged_state_indices.stride(0),
|
||||
staged_state_indices.stride(1),
|
||||
num_spec_decodes,
|
||||
batch_size,
|
||||
NUM_STATE_SLOTS=num_state_slots,
|
||||
BLOCK_STATE_SLOTS=triton.next_power_of_2(num_state_slots),
|
||||
NULL_STATE_ID=NULL_BLOCK_ID,
|
||||
BLOCK_ROWS=BLOCK_ROWS,
|
||||
num_warps=1,
|
||||
launch_pdl=_metadata_launch_pdl(),
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class KimiK3KDAMetadata(GDNAttentionMetadata):
|
||||
pass
|
||||
|
||||
|
||||
class KimiK3KDAMetadataBuilder(GDNAttentionMetadataBuilder):
|
||||
def build( # type: ignore[override]
|
||||
self,
|
||||
common_prefix_len: int,
|
||||
common_attn_metadata: CommonAttentionMetadata,
|
||||
num_accepted_tokens: torch.Tensor | None = None,
|
||||
num_decode_draft_tokens_cpu: torch.Tensor | None = None,
|
||||
fast_build: bool = False,
|
||||
) -> KimiK3KDAMetadata:
|
||||
m = common_attn_metadata
|
||||
query_start_loc = m.query_start_loc
|
||||
query_start_loc_cpu = m.query_start_loc_cpu
|
||||
assert isinstance(self.kv_cache_spec, MambaSpec)
|
||||
# Equivalent PyTorch "align" path:
|
||||
# start = ((seq_lens - 1) // block_size).clamp_(min=0)
|
||||
# offsets = torch.arange(1 + num_speculative_blocks, dtype=torch.int32)
|
||||
# indices = (start[:, None] + offsets).to(torch.int64)
|
||||
# block_table_tensor = torch.gather(block_table, 1, indices)
|
||||
block_table_tensor = _mamba_get_block_table_tensor(
|
||||
m.block_table_tensor,
|
||||
m.seq_lens,
|
||||
self.kv_cache_spec,
|
||||
self.vllm_config.cache_config.mamba_cache_mode,
|
||||
)
|
||||
|
||||
if not self.use_spec_decode or num_decode_draft_tokens_cpu is None:
|
||||
spec_sequence_masks_cpu = None
|
||||
num_spec_decodes = 0
|
||||
else:
|
||||
spec_sequence_masks_cpu = num_decode_draft_tokens_cpu >= 0
|
||||
# A nonnegative entry identifies a spec request. If no draft token
|
||||
# was scheduled, process the whole batch as non-spec instead.
|
||||
if num_decode_draft_tokens_cpu[spec_sequence_masks_cpu].sum().item() == 0:
|
||||
spec_sequence_masks_cpu = None
|
||||
num_spec_decodes = 0
|
||||
else:
|
||||
num_spec_decodes = spec_sequence_masks_cpu.sum().item()
|
||||
|
||||
if num_spec_decodes == 0:
|
||||
# The runner orders ordinary decodes before prefills.
|
||||
num_decodes, num_prefills, num_decode_tokens, num_prefill_tokens = (
|
||||
split_decodes_and_prefills(m, decode_threshold=1)
|
||||
)
|
||||
num_spec_decode_tokens = 0
|
||||
spec_token_indx = None
|
||||
non_spec_token_indx = None
|
||||
spec_state_indices_tensor = None
|
||||
non_spec_state_indices_tensor = block_table_tensor[:, 0]
|
||||
spec_query_start_loc = None
|
||||
non_spec_query_start_loc = query_start_loc
|
||||
non_spec_query_start_loc_cpu = query_start_loc_cpu
|
||||
num_accepted_tokens = None
|
||||
else:
|
||||
assert spec_sequence_masks_cpu is not None
|
||||
assert num_accepted_tokens is not None
|
||||
query_lens_cpu = query_start_loc_cpu.diff()
|
||||
num_query_tokens = query_start_loc_cpu[-1].item()
|
||||
|
||||
# Exclude zero-length cudagraph padding from request-indexed
|
||||
# non-spec metadata.
|
||||
active_non_spec_mask_cpu = (~spec_sequence_masks_cpu) & (query_lens_cpu > 0)
|
||||
non_spec_query_lens_cpu = query_lens_cpu[active_non_spec_mask_cpu]
|
||||
num_non_spec_requests = non_spec_query_lens_cpu.numel()
|
||||
num_non_spec_tokens = non_spec_query_lens_cpu.sum().item()
|
||||
|
||||
# Query length alone cannot distinguish a true decode from a
|
||||
# one-token prefill chunk. Packed decode is safe only when every
|
||||
# active non-spec request is a true, one-token decode.
|
||||
assert m.is_prefilling is not None
|
||||
assert m.is_prefilling.device.type == "cpu"
|
||||
non_spec_is_prefilling = m.is_prefilling[active_non_spec_mask_cpu]
|
||||
use_prefill = torch.any(
|
||||
non_spec_is_prefilling | (non_spec_query_lens_cpu != 1)
|
||||
).item()
|
||||
if use_prefill:
|
||||
num_prefills = num_non_spec_requests
|
||||
num_prefill_tokens = num_non_spec_tokens
|
||||
num_decodes = 0
|
||||
num_decode_tokens = 0
|
||||
else:
|
||||
num_prefills = 0
|
||||
num_prefill_tokens = 0
|
||||
num_decodes = num_non_spec_requests
|
||||
num_decode_tokens = num_non_spec_tokens
|
||||
|
||||
num_spec_decode_tokens = num_query_tokens - num_non_spec_tokens
|
||||
|
||||
if num_prefills == 0 and num_decodes == 0:
|
||||
spec_token_indx = None
|
||||
non_spec_token_indx = None
|
||||
# Real requests precede trailing cudagraph padding.
|
||||
spec_state_indices_tensor = block_table_tensor[
|
||||
:num_spec_decodes, : self.num_spec + 1
|
||||
]
|
||||
non_spec_state_indices_tensor = None
|
||||
# Padding trails real requests, so this prefix already contains
|
||||
# the correct cumulative token counts.
|
||||
spec_query_start_loc = query_start_loc[: num_spec_decodes + 1]
|
||||
non_spec_query_start_loc = None
|
||||
non_spec_query_start_loc_cpu = None
|
||||
num_accepted_tokens = num_accepted_tokens[:num_spec_decodes]
|
||||
else:
|
||||
query_lens = query_start_loc.diff()
|
||||
spec_sequence_masks_gpu = async_tensor_h2d(
|
||||
spec_sequence_masks_cpu, device=query_start_loc.device
|
||||
)
|
||||
spec_token_masks = torch.repeat_interleave(
|
||||
spec_sequence_masks_gpu,
|
||||
query_lens,
|
||||
output_size=num_query_tokens,
|
||||
)
|
||||
# Stable partitioning preserves request-local token order in
|
||||
# both subgroup tensors.
|
||||
index = torch.argsort(spec_token_masks, stable=True)
|
||||
num_non_spec_tokens = num_prefill_tokens + num_decode_tokens
|
||||
non_spec_token_indx = index[:num_non_spec_tokens]
|
||||
spec_token_indx = index[num_non_spec_tokens:]
|
||||
|
||||
# Spec requests carry one state slot per speculative step;
|
||||
# non-spec requests use only their current state slot.
|
||||
spec_state_indices_tensor = block_table_tensor[
|
||||
spec_sequence_masks_cpu, : self.num_spec + 1
|
||||
]
|
||||
non_spec_state_indices_tensor = block_table_tensor[
|
||||
active_non_spec_mask_cpu, 0
|
||||
]
|
||||
|
||||
spec_query_lens = query_lens[spec_sequence_masks_cpu]
|
||||
spec_query_start_loc = torch.zeros(
|
||||
num_spec_decodes + 1,
|
||||
dtype=torch.int32,
|
||||
device=query_start_loc.device,
|
||||
)
|
||||
torch.cumsum(
|
||||
spec_query_lens,
|
||||
dim=0,
|
||||
out=spec_query_start_loc[1:],
|
||||
)
|
||||
if num_prefills > 0:
|
||||
non_spec_query_lens = query_lens[active_non_spec_mask_cpu]
|
||||
non_spec_query_start_loc = torch.zeros(
|
||||
non_spec_query_lens.size(0) + 1,
|
||||
dtype=torch.int32,
|
||||
device=query_start_loc.device,
|
||||
)
|
||||
torch.cumsum(
|
||||
non_spec_query_lens,
|
||||
dim=0,
|
||||
out=non_spec_query_start_loc[1:],
|
||||
)
|
||||
non_spec_query_start_loc_cpu = torch.zeros(
|
||||
non_spec_query_lens_cpu.size(0) + 1,
|
||||
dtype=torch.int32,
|
||||
)
|
||||
torch.cumsum(
|
||||
non_spec_query_lens_cpu,
|
||||
dim=0,
|
||||
out=non_spec_query_start_loc_cpu[1:],
|
||||
)
|
||||
else:
|
||||
# Packed decode consumes one row per request and does not
|
||||
# use cumulative sequence lengths.
|
||||
non_spec_query_start_loc = None
|
||||
non_spec_query_start_loc_cpu = None
|
||||
|
||||
num_accepted_tokens = num_accepted_tokens[spec_sequence_masks_cpu]
|
||||
|
||||
# Unlike the shared GDN layer, Kimi-K3's prefill KDA wrapper prepares
|
||||
# its own chunk indices. Only causal-convolution metadata is needed here.
|
||||
nums_dict, batch_ptr, token_chunk_offset_ptr = None, None, None
|
||||
if num_prefills > 0:
|
||||
has_initial_state = m.compute_num_computed_tokens() > 0
|
||||
if spec_sequence_masks_cpu is not None:
|
||||
has_initial_state = has_initial_state[active_non_spec_mask_cpu]
|
||||
assert non_spec_query_start_loc_cpu is not None
|
||||
nums_dict, batch_ptr, token_chunk_offset_ptr = (
|
||||
compute_causal_conv1d_metadata(
|
||||
non_spec_query_start_loc_cpu,
|
||||
device=query_start_loc.device,
|
||||
)
|
||||
)
|
||||
else:
|
||||
has_initial_state = None
|
||||
|
||||
# Prepare per-request tensors for cudagraph replay. num_actual_tokens
|
||||
# may be token-padded, while state/query/acceptance metadata is indexed
|
||||
# by request.
|
||||
batch_size = m.num_reqs
|
||||
if (
|
||||
self.use_full_cuda_graph
|
||||
and num_spec_decodes > 0
|
||||
and num_prefills == 0
|
||||
and num_decodes == 0
|
||||
and num_spec_decodes <= self.decode_cudagraph_max_bs
|
||||
and num_spec_decode_tokens <= self.decode_cudagraph_max_bs
|
||||
):
|
||||
# Equivalent PyTorch staging:
|
||||
# state[:N].copy_(state_src); state[N:].fill_(NULL_BLOCK_ID)
|
||||
# qsl[:N + 1].copy_(qsl_src); qsl[N + 1:].fill_(qsl_src[-1])
|
||||
# accepted[:N].copy_(accepted_src); accepted[N:].fill_(1)
|
||||
stage_spec_decode_metadata(
|
||||
state_indices=spec_state_indices_tensor,
|
||||
query_start_loc=spec_query_start_loc,
|
||||
num_accepted_tokens=num_accepted_tokens,
|
||||
staged_state_indices=self.spec_state_indices_tensor[:batch_size],
|
||||
staged_query_start_loc=self.spec_query_start_loc[: batch_size + 1],
|
||||
staged_num_accepted_tokens=self.num_accepted_tokens[:batch_size],
|
||||
num_spec_decodes=num_spec_decodes,
|
||||
)
|
||||
|
||||
spec_state_indices_tensor = self.spec_state_indices_tensor[:batch_size]
|
||||
spec_query_start_loc = self.spec_query_start_loc[: batch_size + 1]
|
||||
num_accepted_tokens = self.num_accepted_tokens[:batch_size]
|
||||
|
||||
if (
|
||||
self.use_full_cuda_graph
|
||||
and num_prefills == 0
|
||||
and num_spec_decodes == 0
|
||||
and num_decodes <= self.decode_cudagraph_max_bs
|
||||
):
|
||||
self.non_spec_state_indices_tensor[:num_decodes].copy_(
|
||||
non_spec_state_indices_tensor, non_blocking=True
|
||||
)
|
||||
self.non_spec_state_indices_tensor[num_decodes:batch_size].fill_(
|
||||
NULL_BLOCK_ID
|
||||
)
|
||||
non_spec_state_indices_tensor = self.non_spec_state_indices_tensor[
|
||||
:batch_size
|
||||
]
|
||||
|
||||
return KimiK3KDAMetadata(
|
||||
num_prefills=num_prefills,
|
||||
num_prefill_tokens=num_prefill_tokens,
|
||||
num_decodes=num_decodes,
|
||||
num_decode_tokens=num_decode_tokens,
|
||||
num_spec_decodes=num_spec_decodes,
|
||||
num_spec_decode_tokens=num_spec_decode_tokens,
|
||||
num_actual_tokens=m.num_actual_tokens,
|
||||
has_initial_state=has_initial_state,
|
||||
spec_query_start_loc=spec_query_start_loc,
|
||||
non_spec_query_start_loc=non_spec_query_start_loc,
|
||||
spec_state_indices_tensor=spec_state_indices_tensor,
|
||||
non_spec_state_indices_tensor=non_spec_state_indices_tensor,
|
||||
spec_sequence_masks=None,
|
||||
spec_token_indx=spec_token_indx,
|
||||
non_spec_token_indx=non_spec_token_indx,
|
||||
num_accepted_tokens=num_accepted_tokens,
|
||||
nums_dict=nums_dict,
|
||||
batch_ptr=batch_ptr,
|
||||
token_chunk_offset_ptr=token_chunk_offset_ptr,
|
||||
)
|
||||
|
||||
|
||||
class KimiK3KDAAttentionBackend(GDNAttentionBackend):
|
||||
@staticmethod
|
||||
def get_name() -> str:
|
||||
return "KIMI_K3_KDA"
|
||||
|
||||
@staticmethod
|
||||
def get_builder_cls() -> type[KimiK3KDAMetadataBuilder]:
|
||||
return KimiK3KDAMetadataBuilder
|
||||
@@ -0,0 +1,513 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Kimi-K3 decode GEMM selection for unquantized BF16 on SM103.
|
||||
|
||||
Dispatch is purely by local ``(N, K)`` shape and token count ``M`` — the module
|
||||
name plays no role. Each measured shape maps to a :class:`ProjectionSpec`
|
||||
holding the winning backend per token count. The static part of the decision is
|
||||
resolved once per module at install time into a small ``{M: call}`` plan, so the
|
||||
per-forward path is a single dict lookup.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import Literal
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm import _custom_ops as ops
|
||||
from vllm.model_executor.kernels.linear.cute_dsl.skinny_gemm import (
|
||||
SkinnyGemmConfig,
|
||||
shape_dynamic_skinny_gemm,
|
||||
)
|
||||
from vllm.model_executor.layers.linear import LinearBase, UnquantizedLinearMethod
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
ParallelLMHead,
|
||||
UnquantizedEmbeddingMethod,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
Backend = Literal["cute", "dsv3_fused_a"]
|
||||
# A resolved per-token-count call: the backend plus its CuTe config (None for
|
||||
# dsv3, which needs no config).
|
||||
ResolvedCall = tuple[Backend, SkinnyGemmConfig | None]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ProjectionSpec:
|
||||
n: int
|
||||
k: int
|
||||
dsv3_tokens: frozenset[int] = frozenset()
|
||||
cute_configs: tuple[tuple[int, SkinnyGemmConfig], ...] = ()
|
||||
residual_configs: tuple[tuple[int, SkinnyGemmConfig], ...] = ()
|
||||
name: str = "" # optional debug label; never used for dispatch
|
||||
|
||||
def cute_config(self, num_tokens: int) -> SkinnyGemmConfig | None:
|
||||
return dict(self.cute_configs).get(num_tokens)
|
||||
|
||||
def residual_config(self, num_tokens: int) -> SkinnyGemmConfig | None:
|
||||
return dict(self.residual_configs).get(num_tokens)
|
||||
|
||||
|
||||
def _cute(
|
||||
num_tokens: int,
|
||||
block_size: int,
|
||||
outputs_per_block: int,
|
||||
k_unroll: int,
|
||||
vector_width: int = 8,
|
||||
) -> SkinnyGemmConfig:
|
||||
return SkinnyGemmConfig(
|
||||
num_tokens,
|
||||
block_size,
|
||||
outputs_per_block,
|
||||
k_unroll,
|
||||
vector_width,
|
||||
)
|
||||
|
||||
|
||||
_M1_TO_16 = frozenset(range(1, 17))
|
||||
_M1 = frozenset({1})
|
||||
|
||||
# Keyed by local (N, K). Where two projections share a shape (only 1536x7168:
|
||||
# shared_gate_up_proj and mla_g_proj) the entry is unified.
|
||||
KIMI_K3_PROJECTIONS: dict[tuple[int, int], ProjectionSpec] = {
|
||||
(1536, 128): ProjectionSpec(1536, 128, _M1_TO_16, name="f_b_proj"),
|
||||
(3072, 128): ProjectionSpec(3072, 128, _M1_TO_16, name="f_b_proj"),
|
||||
# 1536x7168 is shared by shared_gate_up_proj and mla_g_proj. dsv3 M1..16 is
|
||||
# only crash-safe once the mla_g aux-stream/PDL capture fix lands (subtask
|
||||
# task_7388aba1); the fallback if it cannot be fixed is dsv3_tokens=_M1.
|
||||
(1536, 7168): ProjectionSpec(
|
||||
1536, 7168, _M1_TO_16, name="shared_gate_up_proj/mla_g_proj"
|
||||
),
|
||||
(3072, 7168): ProjectionSpec(
|
||||
3072,
|
||||
7168,
|
||||
cute_configs=(
|
||||
(1, _cute(1, 224, 3, 4)),
|
||||
(2, _cute(2, 128, 3, 2)),
|
||||
(3, _cute(3, 128, 2, 1)),
|
||||
(4, _cute(4, 64, 2, 2)),
|
||||
(5, _cute(5, 128, 3, 1)),
|
||||
),
|
||||
name="shared_gate_up_proj",
|
||||
),
|
||||
(2112, 7168): ProjectionSpec(2112, 7168, _M1_TO_16, name="fused_qkv_a_proj"),
|
||||
(2304, 1536): ProjectionSpec(2304, 1536, _M1_TO_16, name="q_b_proj"),
|
||||
(4608, 1536): ProjectionSpec(4608, 1536, _M1_TO_16, name="q_b_proj"),
|
||||
(3584, 7168): ProjectionSpec(
|
||||
3584,
|
||||
7168,
|
||||
frozenset(range(2, 9)),
|
||||
((1, _cute(1, 224, 2, 4)),),
|
||||
name="routed_expert_down_proj",
|
||||
),
|
||||
(6288, 7168): ProjectionSpec(
|
||||
6288,
|
||||
7168,
|
||||
cute_configs=(
|
||||
(1, _cute(1, 224, 3, 4)),
|
||||
(2, _cute(2, 64, 3, 2)),
|
||||
(3, _cute(3, 32, 3, 4)),
|
||||
(4, _cute(4, 128, 6, 1)),
|
||||
),
|
||||
name="in_proj_qkvgfab",
|
||||
),
|
||||
(12448, 7168): ProjectionSpec(
|
||||
12448,
|
||||
7168,
|
||||
cute_configs=(
|
||||
(1, _cute(1, 224, 4, 2)),
|
||||
(2, _cute(2, 64, 4, 2)),
|
||||
(3, _cute(3, 64, 2, 2)),
|
||||
),
|
||||
name="in_proj_qkvgfab",
|
||||
),
|
||||
(7168, 768): ProjectionSpec(7168, 768, _M1_TO_16, name="shared_down_proj"),
|
||||
(7168, 1536): ProjectionSpec(
|
||||
7168, 1536, cute_configs=((1, _cute(1, 96, 4, 2)),), name="o_proj"
|
||||
),
|
||||
(7168, 3072): ProjectionSpec(
|
||||
7168,
|
||||
3072,
|
||||
cute_configs=(
|
||||
(1, _cute(1, 96, 2, 4)),
|
||||
(2, _cute(2, 32, 4, 4)),
|
||||
),
|
||||
name="o_proj",
|
||||
),
|
||||
(7168, 3584): ProjectionSpec(
|
||||
7168,
|
||||
3584,
|
||||
cute_configs=(
|
||||
(1, _cute(1, 224, 4, 2)),
|
||||
(2, _cute(2, 64, 4, 2)),
|
||||
),
|
||||
residual_configs=(
|
||||
(1, _cute(1, 64, 4, 2)),
|
||||
(2, _cute(2, 64, 7, 2)),
|
||||
(3, _cute(3, 64, 2, 1)),
|
||||
(4, _cute(4, 64, 2, 1)),
|
||||
),
|
||||
name="routed_expert_up_proj",
|
||||
),
|
||||
(7168, 4224): ProjectionSpec(
|
||||
7168,
|
||||
4224,
|
||||
cute_configs=((1, _cute(1, 96, 4, 2, 4)),),
|
||||
name="dense_down_proj",
|
||||
),
|
||||
(7168, 8448): ProjectionSpec(
|
||||
7168,
|
||||
8448,
|
||||
cute_configs=(
|
||||
(1, _cute(1, 32, 4, 4)),
|
||||
(2, _cute(2, 96, 4, 1)),
|
||||
(3, _cute(3, 96, 4, 1)),
|
||||
),
|
||||
name="dense_down_proj",
|
||||
),
|
||||
(8448, 7168): ProjectionSpec(
|
||||
8448,
|
||||
7168,
|
||||
cute_configs=(
|
||||
(1, _cute(1, 224, 3, 4)),
|
||||
(2, _cute(2, 32, 4, 4)),
|
||||
),
|
||||
name="dense_gate_up_proj",
|
||||
),
|
||||
(16896, 7168): ProjectionSpec(
|
||||
16896,
|
||||
7168,
|
||||
cute_configs=(
|
||||
(1, _cute(1, 224, 6, 4)),
|
||||
(2, _cute(2, 32, 4, 4)),
|
||||
),
|
||||
name="dense_gate_up_proj",
|
||||
),
|
||||
(20480, 7168): ProjectionSpec(
|
||||
20480,
|
||||
7168,
|
||||
cute_configs=(
|
||||
(1, _cute(1, 224, 4, 2)),
|
||||
(2, _cute(2, 64, 4, 2)),
|
||||
(3, _cute(3, 64, 2, 2)),
|
||||
(4, _cute(4, 64, 4, 1)),
|
||||
),
|
||||
name="lm_head",
|
||||
),
|
||||
(40960, 7168): ProjectionSpec(
|
||||
40960,
|
||||
7168,
|
||||
cute_configs=(
|
||||
(1, _cute(1, 128, 4, 2)),
|
||||
(2, _cute(2, 64, 4, 2)),
|
||||
(3, _cute(3, 64, 2, 2)),
|
||||
(4, _cute(4, 64, 4, 1)),
|
||||
),
|
||||
name="lm_head",
|
||||
),
|
||||
# TP16. Measured on B300 over M=1..16 with the same >=5% threshold as the
|
||||
# entries above. The replicated projections (2112x7168, 3584x7168,
|
||||
# 7168x3584) keep their shapes at TP16 and reuse the entries above, and
|
||||
# o_proj lands on 7168x768, which shared_down_proj already covers.
|
||||
(3216, 7168): ProjectionSpec(
|
||||
3216,
|
||||
7168,
|
||||
# Both gaps in this range are measured, not oversights: dsv3 is only
|
||||
# 4% ahead at M6..M8, and at M16 cuBLAS switches to a faster kernel
|
||||
# (11.42us vs dsv3's 11.83us) after trailing it by 6-8% at M9..M15.
|
||||
frozenset(range(9, 16)),
|
||||
cute_configs=(
|
||||
(1, _cute(1, 224, 3, 4)),
|
||||
(2, _cute(2, 128, 4, 2)),
|
||||
(3, _cute(3, 128, 2, 1)),
|
||||
(4, _cute(4, 64, 2, 2)),
|
||||
(5, _cute(5, 128, 3, 1)),
|
||||
),
|
||||
name="in_proj_qkvgfab",
|
||||
),
|
||||
(768, 7168): ProjectionSpec(
|
||||
768,
|
||||
7168,
|
||||
frozenset(range(5, 17)),
|
||||
cute_configs=(
|
||||
(1, _cute(1, 224, 2, 4)),
|
||||
(2, _cute(2, 224, 2, 2)),
|
||||
(3, _cute(3, 224, 2, 2)),
|
||||
(4, _cute(4, 224, 2, 2)),
|
||||
),
|
||||
name="mla_g_proj/shared_gate_up_proj",
|
||||
),
|
||||
(1152, 1536): ProjectionSpec(
|
||||
1152,
|
||||
1536,
|
||||
frozenset(range(2, 17)),
|
||||
((1, _cute(1, 192, 3, 4)),),
|
||||
name="q_b_proj",
|
||||
),
|
||||
(768, 128): ProjectionSpec(768, 128, _M1_TO_16, name="f_b_proj"),
|
||||
# dsv3 drops under 5% from M9 on for this shape.
|
||||
(7168, 384): ProjectionSpec(
|
||||
7168, 384, frozenset(range(1, 9)), name="shared_down_proj"
|
||||
),
|
||||
(4224, 7168): ProjectionSpec(
|
||||
4224,
|
||||
7168,
|
||||
frozenset(range(4, 9)),
|
||||
cute_configs=(
|
||||
(1, _cute(1, 224, 3, 4)),
|
||||
(2, _cute(2, 128, 2, 1)),
|
||||
(3, _cute(3, 64, 2, 2)),
|
||||
),
|
||||
name="dense_gate_up_proj",
|
||||
),
|
||||
(10240, 7168): ProjectionSpec(
|
||||
10240,
|
||||
7168,
|
||||
cute_configs=(
|
||||
(1, _cute(1, 224, 4, 2)),
|
||||
(2, _cute(2, 32, 2, 4)),
|
||||
(3, _cute(3, 64, 4, 1)),
|
||||
(4, _cute(4, 64, 4, 1)),
|
||||
),
|
||||
name="lm_head",
|
||||
),
|
||||
# 7168x2112 (TP16 dense down_proj) has no entry on purpose: K=2112 divides
|
||||
# none of the fused-A tile_k values, and the CuTe kernel is left with
|
||||
# vector_width=2, which measured slower than cuBLAS.
|
||||
}
|
||||
|
||||
|
||||
def _backend_for(
|
||||
spec: ProjectionSpec, num_tokens: int, has_residual: bool
|
||||
) -> Backend | None:
|
||||
if has_residual:
|
||||
return "cute" if spec.residual_config(num_tokens) is not None else None
|
||||
if spec.cute_config(num_tokens) is not None:
|
||||
return "cute"
|
||||
if num_tokens in spec.dsv3_tokens:
|
||||
return "dsv3_fused_a"
|
||||
return None
|
||||
|
||||
|
||||
def select_kimi_k3_backend(
|
||||
num_tokens: int,
|
||||
n: int,
|
||||
k: int,
|
||||
*,
|
||||
has_residual: bool = False,
|
||||
) -> Backend | None:
|
||||
"""Backend for a local ``(N, K)`` at ``num_tokens``, or None to fall back."""
|
||||
spec = KIMI_K3_PROJECTIONS.get((n, k))
|
||||
return _backend_for(spec, num_tokens, has_residual) if spec is not None else None
|
||||
|
||||
|
||||
def _build_plan(spec: ProjectionSpec) -> dict[int, ResolvedCall]:
|
||||
plan: dict[int, ResolvedCall] = {}
|
||||
for num_tokens in range(1, 17):
|
||||
backend = _backend_for(spec, num_tokens, has_residual=False)
|
||||
if backend == "cute":
|
||||
plan[num_tokens] = ("cute", spec.cute_config(num_tokens))
|
||||
elif backend == "dsv3_fused_a":
|
||||
plan[num_tokens] = ("dsv3_fused_a", None)
|
||||
return plan
|
||||
|
||||
|
||||
def _build_residual_plan(spec: ProjectionSpec) -> dict[int, SkinnyGemmConfig]:
|
||||
return {num_tokens: config for num_tokens, config in spec.residual_configs}
|
||||
|
||||
|
||||
def _is_sm103() -> bool:
|
||||
return current_platform.is_device_capability((10, 3))
|
||||
|
||||
|
||||
def _is_packed_row_major(tensor: torch.Tensor) -> bool:
|
||||
return tensor.dim() == 2 and tensor.stride() == (tensor.shape[1], 1)
|
||||
|
||||
|
||||
def _runtime_ok(x: torch.Tensor, weight: torch.Tensor) -> bool:
|
||||
return (
|
||||
_is_packed_row_major(x)
|
||||
and _is_packed_row_major(weight)
|
||||
and x.dtype == torch.bfloat16
|
||||
and weight.dtype == torch.bfloat16
|
||||
and x.is_cuda
|
||||
and weight.is_cuda
|
||||
and x.device == weight.device
|
||||
and x.shape[1] == weight.shape[1]
|
||||
)
|
||||
|
||||
|
||||
def _residual_ok(x: torch.Tensor, weight: torch.Tensor, residual: torch.Tensor) -> bool:
|
||||
return (
|
||||
residual.dim() == 2
|
||||
and residual.dtype == torch.bfloat16
|
||||
and residual.is_cuda
|
||||
and residual.device == x.device
|
||||
and residual.is_contiguous()
|
||||
and residual.shape == (x.shape[0], weight.shape[0])
|
||||
)
|
||||
|
||||
|
||||
def _run_plan(
|
||||
plan: dict[int, ResolvedCall], x: torch.Tensor, weight: torch.Tensor
|
||||
) -> torch.Tensor | None:
|
||||
entry = plan.get(x.shape[0])
|
||||
if entry is None:
|
||||
return None
|
||||
backend, config = entry
|
||||
if backend == "cute":
|
||||
if not shape_dynamic_skinny_gemm.is_available():
|
||||
return None
|
||||
return shape_dynamic_skinny_gemm(x, weight, config, None)
|
||||
if not hasattr(torch.ops._C, "dsv3_fused_a_gemm"):
|
||||
return None
|
||||
output = torch.empty((x.shape[0], weight.shape[0]), dtype=x.dtype, device=x.device)
|
||||
ops.dsv3_fused_a_gemm(output, x, weight.t(), enable_pdl=True)
|
||||
return output
|
||||
|
||||
|
||||
def _run_residual_plan(
|
||||
residual_plan: dict[int, SkinnyGemmConfig],
|
||||
x: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
) -> torch.Tensor | None:
|
||||
config = residual_plan.get(x.shape[0])
|
||||
if config is None or not shape_dynamic_skinny_gemm.is_available():
|
||||
return None
|
||||
return shape_dynamic_skinny_gemm(x, weight, config, residual)
|
||||
|
||||
|
||||
def try_low_latency_gemm(
|
||||
x: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
residual: torch.Tensor | None = None,
|
||||
) -> torch.Tensor | None:
|
||||
"""Run the shape-selected low-latency kernel, or None to fall back.
|
||||
|
||||
Resolves the plan from the shape table on each call; production installs a
|
||||
precomputed plan (see :func:`enable_kimi_k3_low_latency_gemm`) and does not
|
||||
use this path.
|
||||
"""
|
||||
if envs.VLLM_BATCH_INVARIANT or not _is_sm103() or not _runtime_ok(x, weight):
|
||||
return None
|
||||
spec = KIMI_K3_PROJECTIONS.get((weight.shape[0], weight.shape[1]))
|
||||
if spec is None:
|
||||
return None
|
||||
if residual is None:
|
||||
return _run_plan(_build_plan(spec), x, weight)
|
||||
if not _residual_ok(x, weight, residual):
|
||||
return None
|
||||
return _run_residual_plan(_build_residual_plan(spec), x, weight, residual)
|
||||
|
||||
|
||||
class _KimiK3LowLatencyApply:
|
||||
"""Mixin: try the precomputed plan, else defer to the base method."""
|
||||
|
||||
def __init__(self, plan: dict[int, ResolvedCall]) -> None:
|
||||
self._plan = plan
|
||||
|
||||
def apply(
|
||||
self,
|
||||
layer: nn.Module,
|
||||
x: torch.Tensor,
|
||||
bias: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
if (
|
||||
bias is None
|
||||
and not envs.VLLM_BATCH_INVARIANT
|
||||
and _runtime_ok(x, layer.weight)
|
||||
):
|
||||
output = _run_plan(self._plan, x, layer.weight)
|
||||
if output is not None:
|
||||
return output
|
||||
return super().apply(layer, x, bias) # type: ignore[misc]
|
||||
|
||||
|
||||
class KimiK3LowLatencyLinearMethod(_KimiK3LowLatencyApply, UnquantizedLinearMethod):
|
||||
def __init__(
|
||||
self,
|
||||
plan: dict[int, ResolvedCall],
|
||||
residual_plan: dict[int, SkinnyGemmConfig],
|
||||
) -> None:
|
||||
super().__init__(plan)
|
||||
self._residual_plan = residual_plan
|
||||
|
||||
def apply_with_residual(
|
||||
self,
|
||||
layer: nn.Module,
|
||||
x: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
if (
|
||||
not envs.VLLM_BATCH_INVARIANT
|
||||
and _runtime_ok(x, layer.weight)
|
||||
and _residual_ok(x, layer.weight, residual)
|
||||
):
|
||||
output = _run_residual_plan(self._residual_plan, x, layer.weight, residual)
|
||||
if output is not None:
|
||||
return output
|
||||
return torch.addmm(residual, x, layer.weight.t())
|
||||
|
||||
|
||||
class KimiK3LowLatencyEmbeddingMethod(
|
||||
_KimiK3LowLatencyApply, UnquantizedEmbeddingMethod
|
||||
):
|
||||
pass
|
||||
|
||||
|
||||
def enable_kimi_k3_low_latency_gemm(
|
||||
module: nn.Module,
|
||||
dtype: torch.dtype,
|
||||
) -> None:
|
||||
"""Install shape-selected low-latency GEMMs and register CuTe warmups.
|
||||
|
||||
Modules are matched purely by type, an exactly-unquantized method, and a
|
||||
local ``(N, K)`` present in :data:`KIMI_K3_PROJECTIONS`.
|
||||
"""
|
||||
if dtype != torch.bfloat16 or not _is_sm103():
|
||||
return
|
||||
|
||||
warmup_configs: set[SkinnyGemmConfig] = set()
|
||||
residual_warmup_configs: set[SkinnyGemmConfig] = set()
|
||||
for child in module.modules():
|
||||
is_linear = (
|
||||
isinstance(child, LinearBase)
|
||||
and type(child.quant_method) is UnquantizedLinearMethod
|
||||
)
|
||||
# ParallelLMHead is a VocabParallelEmbedding subclass; embed_tokens is
|
||||
# the parent type, so isinstance already excludes it.
|
||||
is_head = (
|
||||
isinstance(child, ParallelLMHead)
|
||||
and type(child.quant_method) is UnquantizedEmbeddingMethod
|
||||
)
|
||||
if not (is_linear or is_head):
|
||||
continue
|
||||
weight = getattr(child, "weight", None)
|
||||
if weight is None or weight.dim() != 2:
|
||||
continue
|
||||
spec = KIMI_K3_PROJECTIONS.get((weight.shape[0], weight.shape[1]))
|
||||
if spec is None:
|
||||
continue
|
||||
if is_linear:
|
||||
child.quant_method = KimiK3LowLatencyLinearMethod(
|
||||
_build_plan(spec), _build_residual_plan(spec)
|
||||
)
|
||||
else:
|
||||
child.quant_method = KimiK3LowLatencyEmbeddingMethod(_build_plan(spec))
|
||||
# Warm up only the configs measured for this module's local (N, K) so a
|
||||
# TP8 deployment does not compile TP4 configs and vice versa.
|
||||
warmup_configs.update(config for _, config in spec.cute_configs)
|
||||
residual_warmup_configs.update(config for _, config in spec.residual_configs)
|
||||
|
||||
if shape_dynamic_skinny_gemm.is_available():
|
||||
if warmup_configs:
|
||||
shape_dynamic_skinny_gemm.request_warmup_configs(dtype, warmup_configs)
|
||||
if residual_warmup_configs:
|
||||
shape_dynamic_skinny_gemm.request_warmup_configs(
|
||||
dtype, residual_warmup_configs, has_residual=True
|
||||
)
|
||||
@@ -0,0 +1,764 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Clean Multi-head Latent Attention for Kimi-K3 (NVIDIA).
|
||||
|
||||
This is a self-contained MLA layer that owns the full attention path:
|
||||
|
||||
hidden_states
|
||||
-> fused pre-attention ops (fused_qkv_a_proj / norms / q_b_proj)
|
||||
-> explicit prefill / decode split
|
||||
prefill: fused key-concat + cache-insert kernel -> run_prefill_new_tokens
|
||||
(+ chunked-context merge); dispatched by cache dtype
|
||||
(bf16 / plain fp8 / fp8_ds_mla)
|
||||
decode : W_UK absorb (BMM1) -> fused q-concat + cache-insert kernel
|
||||
-> impl.forward_mqa -> W_UV up-proj (MQA)
|
||||
-> optional output gate
|
||||
-> o_proj
|
||||
|
||||
Unlike ``MultiHeadLatentAttentionWrapper`` (which delegates orchestration to
|
||||
``MLAAttention.forward``), this class *is* the ``AttentionLayerBase``: it selects
|
||||
the backend, builds the impl, registers itself in the forward context, owns the
|
||||
KV cache, and absorbs ``kv_b_proj`` into ``W_UK_T`` / ``W_UV`` -- mirroring the
|
||||
``DeepseekV4Attention`` structure.
|
||||
|
||||
K3 specifics: optional rotary embedding (disabled for the target model's NoPE
|
||||
layers, enabled for DSpark) and an optional sigmoid output gate (``g_proj``).
|
||||
|
||||
Out of scope (extension points, not wired here): context parallelism (DCP/PCP),
|
||||
sparse/indexer MLA, and the ROCm/aiter fp8/fp4 BMM fast paths.
|
||||
"""
|
||||
|
||||
import math
|
||||
from typing import TYPE_CHECKING, cast
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
|
||||
from vllm.config import CacheConfig, VllmConfig, get_current_vllm_config
|
||||
from vllm.distributed import get_tensor_model_parallel_world_size
|
||||
from vllm.forward_context import get_forward_context
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.attention.attention import (
|
||||
_init_kv_cache_quant,
|
||||
set_default_quant_scales,
|
||||
should_load_quant_weights,
|
||||
)
|
||||
from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.model_executor.layers.linear import (
|
||||
ColumnParallelLinear,
|
||||
MergedColumnParallelLinear,
|
||||
ReplicatedLinear,
|
||||
RowParallelLinear,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
get_and_maybe_dequant_weights,
|
||||
)
|
||||
from vllm.model_executor.layers.rotary_embedding import RotaryEmbedding, get_rope
|
||||
from vllm.model_executor.utils import replace_parameter
|
||||
from vllm.models.common.ops import fused_q_kv_rmsnorm
|
||||
from vllm.models.kimi_k3.nvidia.ops.fused_mla_key_concat_kv_cache import (
|
||||
fused_mla_decode_q_concat_kv_cache_insert,
|
||||
fused_mla_key_concat_ds_mla_insert,
|
||||
fused_mla_key_concat_kv_cache_insert,
|
||||
fused_mla_qkv_quant_kv_cache_fp8_insert,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.transformers_utils.configs.kimi_linear import KimiLinearConfig
|
||||
from vllm.utils.multi_stream_utils import maybe_execute_in_parallel
|
||||
from vllm.utils.torch_utils import (
|
||||
is_quantized_kv_cache,
|
||||
kv_cache_dtype_str_to_dtype,
|
||||
)
|
||||
from vllm.v1.attention.backend import (
|
||||
AttentionBackend,
|
||||
AttentionType,
|
||||
MLAAttentionImpl,
|
||||
)
|
||||
from vllm.v1.attention.backends.mla.prefill import get_mla_prefill_backend
|
||||
from vllm.v1.attention.ops.merge_attn_states import merge_attn_states
|
||||
from vllm.v1.attention.selector import get_attn_backend
|
||||
from vllm.v1.kv_cache_interface import KVCacheSpec, MLAAttentionSpec, get_kv_quant_mode
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.model_executor.layers.attention.mla_attention import MLACommonMetadata
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
# Below this many tokens, overlap the g_proj GEMM on the aux stream with the
|
||||
# attention front-end (the GEMM is small and launch-bound, so the overlap
|
||||
# hides it); at or above it, run the gate on the main stream.
|
||||
_GATE_MULTI_STREAM_TOKEN_THRESHOLD = 512
|
||||
|
||||
|
||||
@torch.compile(backend=current_platform.simple_compile_backend)
|
||||
def _gate_sigmoid_mul(attn_out: torch.Tensor, gate: torch.Tensor) -> torch.Tensor:
|
||||
"""Apply the sigmoid output gate to a precomputed ``g_proj`` projection."""
|
||||
return attn_out * gate.sigmoid()
|
||||
|
||||
|
||||
class MultiHeadLatentAttention(nn.Module, AttentionLayerBase):
|
||||
"""Kimi-K3 Multi-head Latent Attention with optional RoPE and output gate."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: KimiLinearConfig,
|
||||
hidden_size: int,
|
||||
num_heads: int,
|
||||
qk_nope_head_dim: int,
|
||||
qk_rope_head_dim: int,
|
||||
v_head_dim: int,
|
||||
q_lora_rank: int | None,
|
||||
kv_lora_rank: int,
|
||||
use_output_gate: bool = False,
|
||||
cache_config: CacheConfig | None = None,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
prefix: str = "",
|
||||
aux_stream: torch.cuda.Stream | None = None,
|
||||
use_rope: bool = False,
|
||||
non_causal_multi_token_decode: bool = False,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.hidden_size = hidden_size
|
||||
self.qk_nope_head_dim = qk_nope_head_dim
|
||||
self.qk_rope_head_dim = qk_rope_head_dim
|
||||
self.qk_head_dim = qk_nope_head_dim + qk_rope_head_dim
|
||||
self.v_head_dim = v_head_dim
|
||||
self.q_lora_rank = q_lora_rank
|
||||
self.kv_lora_rank = kv_lora_rank
|
||||
self.non_causal_multi_token_decode = non_causal_multi_token_decode
|
||||
# Latent "head" seen by the attention kernel / KV cache.
|
||||
self.head_size = kv_lora_rank + qk_rope_head_dim
|
||||
self.scale = self.qk_head_dim**-0.5
|
||||
self.rms_norm_eps = config.rms_norm_eps
|
||||
self.layer_name = prefix
|
||||
|
||||
self.rotary_emb: RotaryEmbedding | None = None
|
||||
if use_rope:
|
||||
rope_parameters = dict(config.rope_parameters)
|
||||
if rope_parameters["rope_type"] != "default":
|
||||
rope_parameters["rope_type"] = (
|
||||
"deepseek_yarn"
|
||||
if rope_parameters.get("apply_yarn_scaling", True)
|
||||
else "deepseek_llama_scaling"
|
||||
)
|
||||
self.rotary_emb = get_rope(
|
||||
qk_rope_head_dim,
|
||||
max_position=config.max_position_embeddings,
|
||||
rope_parameters=rope_parameters,
|
||||
is_neox_style=False,
|
||||
dtype=torch.float32,
|
||||
)
|
||||
if rope_parameters["rope_type"] == "deepseek_yarn":
|
||||
mscale_all_dim = rope_parameters.get("mscale_all_dim", False)
|
||||
scaling_factor = rope_parameters["factor"]
|
||||
mscale = (
|
||||
1.0
|
||||
if scaling_factor <= 1
|
||||
else 0.1 * float(mscale_all_dim) * math.log(scaling_factor) + 1.0
|
||||
)
|
||||
self.scale *= mscale * mscale
|
||||
# The fused epilogues read the cos/sin table directly in fp32 and run
|
||||
# the RoPE math in fp32, so there is no per-forward dtype cast (and no
|
||||
# precision loss). deepseek_yarn builds cos_sin_cache in fp32 already;
|
||||
# dtype=torch.float32 above forces it for the default rope too (the
|
||||
# DSpark draft, which has no yarn scaling).
|
||||
assert self.rotary_emb.cos_sin_cache.dtype == torch.float32, (
|
||||
"K3 fused MLA RoPE requires an fp32 cos/sin cache; got "
|
||||
f"{self.rotary_emb.cos_sin_cache.dtype}."
|
||||
)
|
||||
|
||||
tp_size = get_tensor_model_parallel_world_size()
|
||||
assert num_heads % tp_size == 0
|
||||
self.num_heads = num_heads
|
||||
self.num_local_heads = num_heads // tp_size
|
||||
|
||||
# ---- Pre-attention projections (fusable front-end) ----
|
||||
# Two query variants: a low-rank q-LoRA path (Kimi-K3) fused with the
|
||||
# kv-down proj, or an uncompressed q path (Kimi-Linear, ``q_lora_rank``
|
||||
# None) with a standalone ``q_proj`` and separate ``kv_a_proj_with_mqa``.
|
||||
if self.q_lora_rank is not None:
|
||||
# Fused q-down + kv-down projection. Replicated (disable_tp) because
|
||||
# the low-rank latents are shared across TP ranks; TP splitting
|
||||
# happens at q_b_proj / kv_b_proj. Checkpoint weights ``q_a_proj``
|
||||
# and ``kv_a_proj_with_mqa`` map onto shards 0 and 1 respectively.
|
||||
self.fused_qkv_a_proj = MergedColumnParallelLinear(
|
||||
self.hidden_size,
|
||||
[self.q_lora_rank, self.kv_lora_rank + self.qk_rope_head_dim],
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.fused_qkv_a_proj",
|
||||
disable_tp=True,
|
||||
)
|
||||
self.q_a_layernorm = RMSNorm(self.q_lora_rank, eps=config.rms_norm_eps)
|
||||
self.q_b_proj = ColumnParallelLinear(
|
||||
self.q_lora_rank,
|
||||
self.num_heads * self.qk_head_dim,
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.q_b_proj",
|
||||
)
|
||||
else:
|
||||
# Uncompressed query: full-rank q_proj (TP-split over heads) plus a
|
||||
# replicated kv-down projection (shared latent across TP ranks).
|
||||
self.q_proj = ColumnParallelLinear(
|
||||
self.hidden_size,
|
||||
self.num_heads * self.qk_head_dim,
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.q_proj",
|
||||
)
|
||||
self.kv_a_proj_with_mqa = ReplicatedLinear(
|
||||
self.hidden_size,
|
||||
self.kv_lora_rank + self.qk_rope_head_dim,
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.kv_a_proj_with_mqa",
|
||||
)
|
||||
self.kv_a_layernorm = RMSNorm(self.kv_lora_rank, eps=config.rms_norm_eps)
|
||||
self.kv_b_proj = ColumnParallelLinear(
|
||||
self.kv_lora_rank,
|
||||
self.num_heads * (self.qk_nope_head_dim + self.v_head_dim),
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.kv_b_proj",
|
||||
)
|
||||
|
||||
# ---- Post-attention projections ----
|
||||
self.use_output_gate = use_output_gate
|
||||
self.g_proj = (
|
||||
ColumnParallelLinear(
|
||||
self.hidden_size,
|
||||
self.num_heads * self.v_head_dim,
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.g_proj",
|
||||
)
|
||||
if use_output_gate
|
||||
else None
|
||||
)
|
||||
# Aux stream (created at the model level, DeepseekV4 convention) for
|
||||
# overlapping the g_proj GEMM with the attention front-end. None on
|
||||
# ROCm/non-cuda -> maybe_execute_in_parallel falls back to sequential.
|
||||
self.aux_stream = aux_stream
|
||||
self._gate_events = (
|
||||
[torch.cuda.Event(), torch.cuda.Event()]
|
||||
if self.g_proj is not None and current_platform.is_cuda_alike()
|
||||
else None
|
||||
)
|
||||
self.o_proj = RowParallelLinear(
|
||||
self.num_heads * self.v_head_dim,
|
||||
self.hidden_size,
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.o_proj",
|
||||
)
|
||||
|
||||
# ---- Attention backend / impl / KV cache ----
|
||||
self.quant_config = quant_config
|
||||
if cache_config is not None:
|
||||
self.kv_cache_dtype = cache_config.cache_dtype
|
||||
else:
|
||||
self.kv_cache_dtype = "auto"
|
||||
|
||||
dtype = torch.get_default_dtype()
|
||||
self.attn_backend = get_attn_backend(
|
||||
self.head_size,
|
||||
dtype,
|
||||
self.kv_cache_dtype,
|
||||
use_mla=True,
|
||||
use_sparse=False,
|
||||
num_heads=self.num_local_heads,
|
||||
)
|
||||
_init_kv_cache_quant(self, quant_config, prefix)
|
||||
# Unit (1.0) scale for the fused fp8 prefill path: q/k/v are cast
|
||||
# unscaled to match forward_mha (the prefill flash path does not
|
||||
# dequantize); only the cache uses _k_scale.
|
||||
self.register_buffer(
|
||||
"_one_scale", torch.ones(1, dtype=torch.float32), persistent=False
|
||||
)
|
||||
|
||||
impl_cls = cast(type[MLAAttentionImpl], self.attn_backend.get_impl_cls())
|
||||
self.impl = impl_cls( # type: ignore[assignment]
|
||||
num_heads=self.num_local_heads,
|
||||
head_size=self.head_size,
|
||||
scale=self.scale,
|
||||
num_kv_heads=1,
|
||||
alibi_slopes=None,
|
||||
sliding_window=None,
|
||||
kv_cache_dtype=self.kv_cache_dtype,
|
||||
logits_soft_cap=None,
|
||||
attn_type=AttentionType.DECODER,
|
||||
kv_sharing_target_layer_name=None,
|
||||
q_lora_rank=self.q_lora_rank,
|
||||
kv_lora_rank=self.kv_lora_rank,
|
||||
qk_nope_head_dim=self.qk_nope_head_dim,
|
||||
qk_rope_head_dim=self.qk_rope_head_dim,
|
||||
qk_head_dim=self.qk_head_dim,
|
||||
v_head_dim=self.v_head_dim,
|
||||
kv_b_proj=self.kv_b_proj,
|
||||
indexer=None,
|
||||
)
|
||||
self.q_pad_num_heads = getattr(self.impl, "q_pad_num_heads", None)
|
||||
|
||||
vllm_config = get_current_vllm_config()
|
||||
parallel_config = vllm_config.parallel_config
|
||||
assert (
|
||||
parallel_config.decode_context_parallel_size <= 1
|
||||
and parallel_config.prefill_context_parallel_size <= 1
|
||||
), "Kimi-K3 MultiHeadLatentAttention does not support context parallelism."
|
||||
self.prefill_backend = get_mla_prefill_backend(vllm_config)(
|
||||
num_heads=self.num_local_heads,
|
||||
scale=self.scale,
|
||||
kv_lora_rank=self.kv_lora_rank,
|
||||
qk_nope_head_dim=self.qk_nope_head_dim,
|
||||
qk_rope_head_dim=self.qk_rope_head_dim,
|
||||
v_head_dim=self.v_head_dim,
|
||||
vllm_config=vllm_config,
|
||||
)
|
||||
|
||||
compilation_config = vllm_config.compilation_config
|
||||
if prefix in compilation_config.static_forward_context:
|
||||
raise ValueError(f"Duplicate layer name: {prefix}")
|
||||
compilation_config.static_forward_context[prefix] = self
|
||||
self.kv_cache = torch.tensor([])
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# AttentionLayerBase interface
|
||||
# ------------------------------------------------------------------
|
||||
def get_attn_backend(self) -> type[AttentionBackend]:
|
||||
return self.attn_backend
|
||||
|
||||
def get_kv_cache_spec(self, vllm_config: VllmConfig) -> KVCacheSpec:
|
||||
kv_cache_dtype = kv_cache_dtype_str_to_dtype(
|
||||
self.kv_cache_dtype, vllm_config.model_config
|
||||
)
|
||||
# TODO: Remove this mypy workaround once the K3 PR is fully merged.
|
||||
return MLAAttentionSpec( # type: ignore[call-arg]
|
||||
block_size=vllm_config.cache_config.block_size,
|
||||
num_kv_heads=1,
|
||||
head_size=self.head_size,
|
||||
dtype=kv_cache_dtype,
|
||||
cache_dtype_str=self.kv_cache_dtype,
|
||||
kv_quant_mode=get_kv_quant_mode(self.kv_cache_dtype),
|
||||
non_causal_multi_token_decode=self.non_causal_multi_token_decode,
|
||||
)
|
||||
|
||||
def process_weights_after_loading(self, act_dtype: torch.dtype) -> None:
|
||||
"""Absorb ``kv_b_proj`` into decode-time ``W_UK_T`` / ``W_UV`` bmm weights.
|
||||
|
||||
``kv_b_proj`` produces ``[k_nope; v]`` per head from the ``kv_lora_rank``
|
||||
latent. For the MQA decode path we pre-split it so that queries are
|
||||
projected into latent space by ``W_UK_T`` and the attention output is
|
||||
projected back to ``v`` by ``W_UV`` -- avoiding materializing full K/V.
|
||||
"""
|
||||
kv_b_proj_weight = get_and_maybe_dequant_weights(
|
||||
self.kv_b_proj, out_dtype=act_dtype
|
||||
).T
|
||||
assert kv_b_proj_weight.shape == (
|
||||
self.kv_lora_rank,
|
||||
self.num_local_heads * (self.qk_nope_head_dim + self.v_head_dim),
|
||||
), f"{kv_b_proj_weight.shape=}"
|
||||
kv_b_proj_weight = kv_b_proj_weight.view(
|
||||
self.kv_lora_rank,
|
||||
self.num_local_heads,
|
||||
self.qk_nope_head_dim + self.v_head_dim,
|
||||
)
|
||||
W_UK, W_UV = kv_b_proj_weight.split(
|
||||
[self.qk_nope_head_dim, self.v_head_dim], dim=-1
|
||||
)
|
||||
# (L, N, V) -> (N, L, V)
|
||||
replace_parameter(self, "W_UV", W_UV.transpose(0, 1), prefer_copy=True)
|
||||
# (L, N, P) -> (N, P, L)
|
||||
replace_parameter(self, "W_UK_T", W_UK.permute(1, 2, 0), prefer_copy=True)
|
||||
|
||||
quant_method = (
|
||||
self.quant_config.get_quant_method(self, prefix=self.layer_name)
|
||||
if self.quant_config
|
||||
else None
|
||||
)
|
||||
if not should_load_quant_weights(quant_method):
|
||||
set_default_quant_scales(self, register_buffer=False)
|
||||
|
||||
# Precompute reciprocal scales once here (scales are final after load;
|
||||
# K3 has no runtime calculate_kv_scales path) so the fp8 fused kernels
|
||||
# in the decode/prefill hot path take a ready inverse instead of
|
||||
# launching a per-step reciprocal kernel.
|
||||
self.register_buffer(
|
||||
"_q_scale_inv", self._q_scale.reciprocal().reshape(1), persistent=False
|
||||
)
|
||||
self.register_buffer(
|
||||
"_k_scale_inv", self._k_scale.reciprocal().reshape(1), persistent=False
|
||||
)
|
||||
|
||||
def _v_up_proj(self, x: torch.Tensor, out: torch.Tensor) -> None:
|
||||
"""Project latent attention output back to ``v`` via ``W_UV`` (bmm)."""
|
||||
# (B, N, L) -> (N, B, L)
|
||||
x = x.view(-1, self.num_local_heads, self.kv_lora_rank).transpose(0, 1)
|
||||
out = out.view(-1, self.num_local_heads, self.v_head_dim)
|
||||
# (N, B, L) x (N, L, V) -> (N, B, V) written transposed into (B, N, V)
|
||||
torch.bmm(x, self.W_UV, out=out.transpose(0, 1))
|
||||
|
||||
def _attn_read_kv_cache(self) -> torch.Tensor:
|
||||
"""Latent cache as seen by the attention read kernels (decode / context).
|
||||
|
||||
A plain per-tensor fp8 cache is stored as ``uint8``; view it as fp8 so
|
||||
the backend reads it as E4M3 rather than fp4/E2M1 -- the latter doubles
|
||||
the perceived head dim (``head_size * 2``) and fails the kernel's
|
||||
``head_dim_k == head_dim_q`` check. Mirrors ``MLAAttention.forward``;
|
||||
the fp8_ds_mla layout keeps its native uint8 view.
|
||||
"""
|
||||
cache = self.kv_cache
|
||||
if (
|
||||
is_quantized_kv_cache(self.kv_cache_dtype)
|
||||
and self.kv_cache_dtype != "fp8_ds_mla"
|
||||
):
|
||||
return cache.view(current_platform.fp8_dtype())
|
||||
return cache
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Forward
|
||||
# ------------------------------------------------------------------
|
||||
def _forward_attn(
|
||||
self,
|
||||
positions: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
"""Attention front-end: fused qkv-a proj -> norms -> q_b -> attention.
|
||||
|
||||
Returns the pre-gate attention output ``[num_tokens,
|
||||
num_local_heads * v_head_dim]``. On a profile/dummy run
|
||||
it returns a zeroed buffer.
|
||||
"""
|
||||
if self.q_lora_rank is not None:
|
||||
qkv_lora = self.fused_qkv_a_proj(hidden_states)[0]
|
||||
q_c, kv_c, k_pe = qkv_lora.split(
|
||||
[self.q_lora_rank, self.kv_lora_rank, self.qk_rope_head_dim], dim=-1
|
||||
)
|
||||
q_c, kv_c_normed = fused_q_kv_rmsnorm(
|
||||
q_c,
|
||||
kv_c,
|
||||
self.q_a_layernorm.weight.data,
|
||||
self.kv_a_layernorm.weight.data,
|
||||
self.rms_norm_eps,
|
||||
)
|
||||
q = self.q_b_proj(q_c)[0].view(-1, self.num_local_heads, self.qk_head_dim)
|
||||
else:
|
||||
# Uncompressed query: project directly (no q-LoRA, no q norm) and
|
||||
# normalize only the kv latent.
|
||||
q = self.q_proj(hidden_states)[0].view(
|
||||
-1, self.num_local_heads, self.qk_head_dim
|
||||
)
|
||||
kv_lora = self.kv_a_proj_with_mqa(hidden_states)[0]
|
||||
kv_c, k_pe = kv_lora.split(
|
||||
[self.kv_lora_rank, self.qk_rope_head_dim], dim=-1
|
||||
)
|
||||
kv_c_normed = self.kv_a_layernorm(kv_c)
|
||||
k_pe = k_pe.unsqueeze(1)
|
||||
|
||||
attn_out = torch.empty(
|
||||
(hidden_states.shape[0], self.num_local_heads * self.v_head_dim),
|
||||
dtype=hidden_states.dtype,
|
||||
device=hidden_states.device,
|
||||
)
|
||||
self._attention(positions, q, kv_c_normed, k_pe, attn_out)
|
||||
return attn_out
|
||||
|
||||
def forward(
|
||||
self,
|
||||
positions: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
# Both branches produce (attn_out, gate); they differ only in whether
|
||||
# the g_proj GEMM is overlapped on the aux stream.
|
||||
g_proj = self.g_proj
|
||||
events = self._gate_events
|
||||
if (
|
||||
g_proj is not None
|
||||
and events is not None
|
||||
and self.aux_stream is not None
|
||||
and hidden_states.shape[0] < _GATE_MULTI_STREAM_TOKEN_THRESHOLD
|
||||
):
|
||||
attn_out, gate = maybe_execute_in_parallel(
|
||||
lambda: self._forward_attn(positions, hidden_states),
|
||||
lambda: g_proj(hidden_states)[0],
|
||||
events[0],
|
||||
events[1],
|
||||
self.aux_stream,
|
||||
)
|
||||
else:
|
||||
attn_out = self._forward_attn(positions, hidden_states)
|
||||
gate = g_proj(hidden_states)[0] if g_proj is not None else None
|
||||
|
||||
if gate is not None:
|
||||
attn_out = _gate_sigmoid_mul(attn_out, gate)
|
||||
|
||||
# ``o_proj`` (RowParallelLinear + out-of-place all-reduce) returns a
|
||||
# fresh private tensor, so return it directly rather than copying into a
|
||||
# caller buffer -- the previous ``output[:] = ...`` convention forced an
|
||||
# extra [num_tokens, hidden] copy per layer.
|
||||
return self.o_proj(attn_out)[0]
|
||||
|
||||
@eager_break_during_capture
|
||||
def _attention(
|
||||
self,
|
||||
positions: torch.Tensor,
|
||||
q: torch.Tensor,
|
||||
kv_c_normed: torch.Tensor,
|
||||
k_pe: torch.Tensor,
|
||||
attn_out: torch.Tensor,
|
||||
) -> None:
|
||||
forward_context = get_forward_context()
|
||||
attn_metadata_by_layer = forward_context.attn_metadata
|
||||
if attn_metadata_by_layer is None:
|
||||
attn_out.zero_()
|
||||
return
|
||||
assert isinstance(attn_metadata_by_layer, dict)
|
||||
attn_metadata = cast(
|
||||
"MLACommonMetadata", attn_metadata_by_layer[self.layer_name]
|
||||
)
|
||||
|
||||
num_actual_toks = attn_metadata.num_actual_tokens
|
||||
slot_mapping_by_layer = forward_context.slot_mapping
|
||||
assert isinstance(slot_mapping_by_layer, dict)
|
||||
slot_mapping = slot_mapping_by_layer[self.layer_name]
|
||||
|
||||
q = q[:num_actual_toks]
|
||||
kv_c_normed = kv_c_normed[:num_actual_toks]
|
||||
k_pe = k_pe[:num_actual_toks]
|
||||
positions = positions[:num_actual_toks]
|
||||
attn_out = attn_out[:num_actual_toks]
|
||||
|
||||
cos_sin_cache = None
|
||||
rope_positions = None
|
||||
if self.rotary_emb is not None:
|
||||
# Pass the fp32 cos/sin table straight to the fused epilogue (it reads
|
||||
# fp32 and does the RoPE math in fp32) -- no per-forward dtype cast.
|
||||
cos_sin_cache = self.rotary_emb.cos_sin_cache
|
||||
rope_positions = positions
|
||||
|
||||
# Decode tokens are laid out first, prefill tokens after. The fused
|
||||
# prefill covers every supported config (bf16 / plain-fp8 /
|
||||
# fp8_ds_mla), so there is no dense-MHA (forward_mha) fallback.
|
||||
num_mqa_tokens = attn_metadata.num_decode_tokens
|
||||
num_mha_tokens = q.size(0) - num_mqa_tokens
|
||||
|
||||
# Both the prefill and decode fused epilogues write their own cache
|
||||
# slice, so there is no separate do_kv_cache_update.
|
||||
|
||||
# ---- Prefill: fused key-concat + cache-insert + attention ----
|
||||
if num_mha_tokens > 0:
|
||||
self._forward_prefill_fused(
|
||||
q[num_mqa_tokens:],
|
||||
kv_c_normed[num_mqa_tokens:],
|
||||
k_pe[num_mqa_tokens:],
|
||||
rope_positions[num_mqa_tokens:] if rope_positions is not None else None,
|
||||
cos_sin_cache,
|
||||
slot_mapping[num_mqa_tokens:num_actual_toks],
|
||||
attn_metadata,
|
||||
attn_out[num_mqa_tokens:],
|
||||
)
|
||||
|
||||
# ---- Decode: latent multi-query attention ----
|
||||
if num_mqa_tokens > 0:
|
||||
mqa_q_nope, mqa_q_pe = q[:num_mqa_tokens].split(
|
||||
[self.qk_nope_head_dim, self.qk_rope_head_dim], dim=-1
|
||||
)
|
||||
# BMM1: absorb q_nope into latent space. (N,B,P) x (N,P,L) -> (B,N,L)
|
||||
ql_nope = torch.bmm(mqa_q_nope.transpose(0, 1), self.W_UK_T).transpose(0, 1)
|
||||
# Fused: concat mqa_q = [ql_nope | q_pe] and insert the decode-token
|
||||
# latent into the paged cache (one launch, right before forward_mqa).
|
||||
mqa_q = self._decode_concat_cache(
|
||||
ql_nope,
|
||||
mqa_q_pe,
|
||||
kv_c_normed[:num_mqa_tokens],
|
||||
k_pe[:num_mqa_tokens],
|
||||
rope_positions[:num_mqa_tokens] if rope_positions is not None else None,
|
||||
cos_sin_cache,
|
||||
slot_mapping[:num_mqa_tokens],
|
||||
)
|
||||
latent_out, _lse = self.impl.forward_mqa( # type: ignore[attr-defined]
|
||||
mqa_q, self._attn_read_kv_cache(), attn_metadata, self
|
||||
)
|
||||
self._v_up_proj(latent_out, out=attn_out[:num_mqa_tokens])
|
||||
|
||||
def _decode_concat_cache(
|
||||
self,
|
||||
ql_nope: torch.Tensor,
|
||||
q_pe: torch.Tensor,
|
||||
kv_c_normed: torch.Tensor,
|
||||
k_pe: torch.Tensor,
|
||||
positions: torch.Tensor | None,
|
||||
cos_sin_cache: torch.Tensor | None,
|
||||
slot_mapping: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
"""Fused decode query-concat + latent cache insert, dispatched by cache
|
||||
dtype (same policy as prefill: fp8 cache -> fp8 query)."""
|
||||
if self.kv_cache_dtype == "fp8_ds_mla":
|
||||
cache = self.kv_cache
|
||||
if cache.dtype != torch.uint8:
|
||||
cache = cache.view(torch.uint8)
|
||||
return fused_mla_decode_q_concat_kv_cache_insert(
|
||||
ql_nope,
|
||||
q_pe,
|
||||
kv_c_normed,
|
||||
k_pe,
|
||||
cache,
|
||||
slot_mapping,
|
||||
ds_mla=True,
|
||||
positions=positions,
|
||||
cos_sin_cache=cos_sin_cache,
|
||||
)
|
||||
if is_quantized_kv_cache(self.kv_cache_dtype):
|
||||
assert self.impl.supports_quant_query_input, ( # type: ignore[attr-defined]
|
||||
"Kimi-K3 fp8 KV cache decode requires a backend that accepts an "
|
||||
"fp8 (quantized) query input."
|
||||
)
|
||||
cache = self.kv_cache
|
||||
if cache.dtype != torch.float8_e4m3fn:
|
||||
cache = cache.view(torch.float8_e4m3fn)
|
||||
return fused_mla_decode_q_concat_kv_cache_insert(
|
||||
ql_nope,
|
||||
q_pe,
|
||||
kv_c_normed,
|
||||
k_pe,
|
||||
cache,
|
||||
slot_mapping,
|
||||
q_scale_inv=self._q_scale_inv,
|
||||
cache_scale_inv=self._k_scale_inv,
|
||||
positions=positions,
|
||||
cos_sin_cache=cos_sin_cache,
|
||||
)
|
||||
return fused_mla_decode_q_concat_kv_cache_insert(
|
||||
ql_nope,
|
||||
q_pe,
|
||||
kv_c_normed,
|
||||
k_pe,
|
||||
self.kv_cache,
|
||||
slot_mapping,
|
||||
positions=positions,
|
||||
cos_sin_cache=cos_sin_cache,
|
||||
)
|
||||
|
||||
def _forward_prefill_fused(
|
||||
self,
|
||||
q: torch.Tensor,
|
||||
kv_c_normed: torch.Tensor,
|
||||
k_pe: torch.Tensor,
|
||||
positions: torch.Tensor | None,
|
||||
cos_sin_cache: torch.Tensor | None,
|
||||
slot_mapping: torch.Tensor,
|
||||
attn_metadata,
|
||||
out: torch.Tensor,
|
||||
) -> None:
|
||||
"""Prefill using the fused key-concat + cache-insert kernel.
|
||||
|
||||
Replaces ``_concat_k_nope_k_pe`` and the prefill cache write with one
|
||||
fused kernel launch, dispatched by cache dtype. The chunked context
|
||||
gather + online-softmax merge are delegated to the impl.
|
||||
|
||||
Supported configs (K3 fp8 policy):
|
||||
- bf16 cache -> bf16 prefill query
|
||||
- plain fp8 cache -> fp8 prefill query (unscaled q/k/v; cache _k_scale)
|
||||
- fp8_ds_mla cache -> bf16 prefill query (656B per-tile self-scaled)
|
||||
"""
|
||||
prefill = attn_metadata.prefill
|
||||
has_context = prefill.chunked_context is not None
|
||||
fp8_prefill = prefill.q_data_type == current_platform.fp8_dtype()
|
||||
|
||||
kv_nope = self.kv_b_proj(kv_c_normed)[0].view(
|
||||
-1, self.num_local_heads, self.qk_nope_head_dim + self.v_head_dim
|
||||
)
|
||||
k_nope, v = kv_nope.split([self.qk_nope_head_dim, self.v_head_dim], dim=-1)
|
||||
|
||||
if self.kv_cache_dtype == "fp8_ds_mla":
|
||||
# fp8_ds_mla cache (656B, per-tile self-scaled); bf16 attention.
|
||||
assert not fp8_prefill, (
|
||||
"Kimi-K3 fp8_ds_mla uses a bf16 prefill query; fp8 prefill "
|
||||
"query is not supported with fp8_ds_mla."
|
||||
)
|
||||
kv_cache = self.kv_cache
|
||||
if kv_cache.dtype != torch.uint8:
|
||||
kv_cache = kv_cache.view(torch.uint8)
|
||||
k = fused_mla_key_concat_ds_mla_insert(
|
||||
q,
|
||||
k_nope,
|
||||
k_pe,
|
||||
kv_c_normed,
|
||||
kv_cache,
|
||||
slot_mapping,
|
||||
positions,
|
||||
cos_sin_cache,
|
||||
)
|
||||
elif is_quantized_kv_cache(self.kv_cache_dtype):
|
||||
assert fp8_prefill, (
|
||||
"Kimi-K3 fp8 KV cache requires an fp8 prefill query; enable "
|
||||
"--attention-config '{\"use_prefill_query_quantization\": true}'."
|
||||
)
|
||||
# Plain per-tensor fp8: quant q/k/v (unscaled, matching forward_mha's
|
||||
# unscaled `.to(fp8)`) and insert the fp8 latent (scaled by _k_scale).
|
||||
kv_cache = self.kv_cache
|
||||
if kv_cache.dtype != torch.float8_e4m3fn:
|
||||
kv_cache = kv_cache.view(torch.float8_e4m3fn)
|
||||
q, k, v = fused_mla_qkv_quant_kv_cache_fp8_insert(
|
||||
q,
|
||||
k_nope,
|
||||
k_pe,
|
||||
kv_c_normed,
|
||||
v,
|
||||
kv_cache,
|
||||
slot_mapping,
|
||||
self._one_scale,
|
||||
self._one_scale,
|
||||
self._one_scale,
|
||||
self._k_scale_inv,
|
||||
positions,
|
||||
cos_sin_cache,
|
||||
)
|
||||
else:
|
||||
# Concat full K = [k_nope | k_pe] and insert [kv_c_normed | k_pe]
|
||||
# into the paged cache for these prefill tokens, in one launch.
|
||||
k = fused_mla_key_concat_kv_cache_insert(
|
||||
q,
|
||||
k_nope,
|
||||
k_pe,
|
||||
kv_c_normed,
|
||||
self.kv_cache,
|
||||
slot_mapping,
|
||||
positions,
|
||||
cos_sin_cache,
|
||||
)
|
||||
|
||||
# When there is no chunked context, backends that honor `out` write the
|
||||
# attention result straight into it, avoiding a slice+flatten+copy.
|
||||
writes_out = not has_context and prefill.prefill_backend.supports_out()
|
||||
output_prefill = prefill.prefill_backend.run_prefill_new_tokens(
|
||||
q=q,
|
||||
k=k,
|
||||
v=v,
|
||||
return_softmax_lse=has_context,
|
||||
out=(
|
||||
out.view(-1, self.num_local_heads, self.v_head_dim)
|
||||
if writes_out
|
||||
else None
|
||||
),
|
||||
)
|
||||
|
||||
if has_context:
|
||||
context_output, context_lse = self.impl._compute_prefill_context( # type: ignore[attr-defined]
|
||||
q, self._attn_read_kv_cache(), attn_metadata, self._k_scale
|
||||
)
|
||||
suffix_output, suffix_lse = output_prefill
|
||||
out = out.view(-1, self.num_local_heads, self.v_head_dim)
|
||||
merge_attn_states(
|
||||
output=out,
|
||||
prefix_output=context_output[..., : self.v_head_dim],
|
||||
prefix_lse=context_lse,
|
||||
suffix_output=suffix_output[..., : self.v_head_dim],
|
||||
suffix_lse=suffix_lse,
|
||||
prefill_tokens_with_context=prefill.chunked_context.prefill_tokens_with_context,
|
||||
)
|
||||
elif not writes_out:
|
||||
out.copy_(output_prefill[..., : self.v_head_dim].flatten(start_dim=-2))
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,443 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Inference-only Kimi-K3 Multi-Token-Prediction (MTP) draft model."""
|
||||
|
||||
import copy
|
||||
from collections.abc import Iterable
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.forward_context import get_forward_context, is_forward_context_available
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
fused_moe_make_expert_params_mapping,
|
||||
)
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
ParallelLMHead,
|
||||
VocabParallelEmbedding,
|
||||
)
|
||||
from vllm.model_executor.model_loader.weight_utils import (
|
||||
default_weight_loader,
|
||||
maybe_remap_kv_scale_name,
|
||||
)
|
||||
from vllm.model_executor.models.utils import get_pp_missing_layer_names, maybe_prefix
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.transformers_utils.configs.kimi_linear import KimiLinearConfig
|
||||
|
||||
from ..common.mtp import fused_mtp_input
|
||||
from .low_latency_gemm import enable_kimi_k3_low_latency_gemm
|
||||
from .model import (
|
||||
KimiDecoderLayer,
|
||||
KimiMoE,
|
||||
get_spec_layer_idx_from_weight_name,
|
||||
make_kimi_k3_mega_moe_expert_params_mapping,
|
||||
)
|
||||
from .ops.sequence_parallel import sp_all_gather, sp_padding_mask, sp_shard
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class SharedHead(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
config: KimiLinearConfig,
|
||||
prefix: str,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
self.head = ParallelLMHead(
|
||||
config.vocab_size,
|
||||
config.hidden_size,
|
||||
quant_config=quant_config,
|
||||
prefix=maybe_prefix(prefix, "head"),
|
||||
)
|
||||
|
||||
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
||||
return self.norm(hidden_states)
|
||||
|
||||
|
||||
class KimiK3MultiTokenPredictorLayer(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
config: KimiLinearConfig,
|
||||
vllm_config: VllmConfig,
|
||||
prefix: str,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.config = config
|
||||
quant_config = vllm_config.quant_config
|
||||
|
||||
self.enorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
self.hnorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
self.eh_proj = nn.Linear(config.hidden_size * 2, config.hidden_size, bias=False)
|
||||
|
||||
self.shared_head = SharedHead(
|
||||
config=config, prefix=prefix, quant_config=quant_config
|
||||
)
|
||||
# The MTP block starts without the base model's AttnRes state.
|
||||
block_config = copy.copy(config)
|
||||
block_config.attn_res_block_size = None
|
||||
# NOTE: the prefix must end in the numeric spec-layer index so that
|
||||
# KimiDecoderLayer can parse ``layer_idx`` and pick MLA (full attn).
|
||||
# Own aux stream for the MLA g_proj output-gate overlap (DeepseekV4
|
||||
# convention: the MTP block creates its own stream).
|
||||
aux_stream = torch.cuda.Stream()
|
||||
self.mtp_block = KimiDecoderLayer(
|
||||
block_config, vllm_config, prefix=prefix, aux_stream=aux_stream
|
||||
)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
previous_hidden_states: torch.Tensor,
|
||||
inputs_embeds: torch.Tensor | None = None,
|
||||
spec_step_index: int = 0,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
assert inputs_embeds is not None
|
||||
hidden_states = self.eh_proj(
|
||||
fused_mtp_input(
|
||||
positions,
|
||||
inputs_embeds,
|
||||
previous_hidden_states,
|
||||
self.enorm.weight,
|
||||
self.hnorm.weight,
|
||||
self.enorm.variance_epsilon,
|
||||
)
|
||||
)
|
||||
if self.mtp_block.use_sequence_parallel:
|
||||
if envs.VLLM_MOE_SKIP_PADDING and is_forward_context_available():
|
||||
forward_context = get_forward_context()
|
||||
forward_context.is_padding = sp_padding_mask(
|
||||
forward_context.is_padding, hidden_states
|
||||
)
|
||||
hidden_states = sp_shard(hidden_states)
|
||||
|
||||
hidden_states, _, residual = self.mtp_block(
|
||||
positions=positions,
|
||||
hidden_states=hidden_states,
|
||||
residual=None,
|
||||
)
|
||||
if self.mtp_block.use_sequence_parallel:
|
||||
assert residual is not None
|
||||
hidden_states = hidden_states + residual
|
||||
hidden_states = sp_all_gather(hidden_states)[: positions.shape[0]]
|
||||
logits_hidden_states = self.shared_head.norm(hidden_states)
|
||||
return logits_hidden_states, hidden_states
|
||||
|
||||
# Produce the normalized logits input and the pre-norm recurrent state
|
||||
# in one fused add-RMSNorm launch.
|
||||
logits_hidden_states, hidden_states = self.shared_head.norm(
|
||||
hidden_states, residual
|
||||
)
|
||||
return logits_hidden_states, hidden_states
|
||||
|
||||
|
||||
class KimiK3MultiTokenPredictor(nn.Module):
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
super().__init__()
|
||||
config: KimiLinearConfig = vllm_config.model_config.hf_text_config
|
||||
self.config = config
|
||||
self.mtp_start_layer_idx = config.num_hidden_layers
|
||||
self.num_mtp_layers = config.num_nextn_predict_layers
|
||||
|
||||
self.layers = torch.nn.ModuleDict(
|
||||
{
|
||||
str(idx): KimiK3MultiTokenPredictorLayer(
|
||||
config, vllm_config, f"{prefix}.layers.{idx}"
|
||||
)
|
||||
for idx in range(
|
||||
self.mtp_start_layer_idx,
|
||||
self.mtp_start_layer_idx + self.num_mtp_layers,
|
||||
)
|
||||
}
|
||||
)
|
||||
self.embed_tokens = VocabParallelEmbedding(
|
||||
config.vocab_size,
|
||||
config.hidden_size,
|
||||
prefix=maybe_prefix(prefix, "embed_tokens"),
|
||||
)
|
||||
self.logits_processor = LogitsProcessor(config.vocab_size)
|
||||
|
||||
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
|
||||
return self.embed_tokens(input_ids)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
previous_hidden_states: torch.Tensor,
|
||||
inputs_embeds: torch.Tensor | None = None,
|
||||
spec_step_idx: int = 0,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
if inputs_embeds is None:
|
||||
inputs_embeds = self.embed_tokens(input_ids)
|
||||
current_step_idx = spec_step_idx % self.num_mtp_layers
|
||||
return self.layers[str(self.mtp_start_layer_idx + current_step_idx)](
|
||||
input_ids,
|
||||
positions,
|
||||
previous_hidden_states,
|
||||
inputs_embeds,
|
||||
current_step_idx,
|
||||
)
|
||||
|
||||
def compute_logits(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
spec_step_idx: int = 0,
|
||||
) -> torch.Tensor:
|
||||
current_step_idx = spec_step_idx % self.num_mtp_layers
|
||||
mtp_layer = self.layers[str(self.mtp_start_layer_idx + current_step_idx)]
|
||||
logits = self.logits_processor(mtp_layer.shared_head.head, hidden_states)
|
||||
return logits
|
||||
|
||||
|
||||
class KimiK3MTP(nn.Module):
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
super().__init__()
|
||||
self.config = vllm_config.model_config.hf_text_config
|
||||
self.quant_config = vllm_config.quant_config
|
||||
self.model = KimiK3MultiTokenPredictor(
|
||||
vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model")
|
||||
)
|
||||
enable_kimi_k3_low_latency_gemm(self, vllm_config.model_config.dtype)
|
||||
|
||||
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
|
||||
return self.model.embed_input_ids(input_ids)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.Tensor | None,
|
||||
positions: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
intermediate_tensors: IntermediateTensors | None = None,
|
||||
inputs_embeds: torch.Tensor | None = None,
|
||||
spec_step_idx: int = 0,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
return self.model(
|
||||
input_ids,
|
||||
positions,
|
||||
hidden_states,
|
||||
inputs_embeds,
|
||||
spec_step_idx,
|
||||
)
|
||||
|
||||
def compute_logits(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
spec_step_idx: int = 0,
|
||||
) -> torch.Tensor | None:
|
||||
return self.model.compute_logits(hidden_states, spec_step_idx)
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
# Mirror KimiLinearForCausalLM.load_weights naming: leading-dot shard
|
||||
# names, q_lora-conditional fused QKV, and w1/w2/w3 expert weights.
|
||||
kda_config = self.config.linear_attn_config
|
||||
use_full_rank_gate = bool(
|
||||
kda_config and kda_config.get("use_full_rank_gate", False)
|
||||
)
|
||||
beta_shard_id = 5 if use_full_rank_gate else 3
|
||||
stacked_params_mapping = [
|
||||
# (param_name, shard_name, shard_id)
|
||||
(".in_proj_qkvgfab", ".q_proj", 0),
|
||||
(".in_proj_qkvgfab", ".k_proj", 1),
|
||||
(".in_proj_qkvgfab", ".v_proj", 2),
|
||||
(".in_proj_qkvgfab", ".b_proj", beta_shard_id),
|
||||
(".in_proj_qkvgfab", ".f_a_proj", 4),
|
||||
(".conv1d", ".q_conv1d", 0),
|
||||
(".conv1d", ".k_conv1d", 1),
|
||||
(".conv1d", ".v_conv1d", 2),
|
||||
(".gate_up_proj", ".gate_proj", 0),
|
||||
(".gate_up_proj", ".up_proj", 1),
|
||||
]
|
||||
if use_full_rank_gate:
|
||||
stacked_params_mapping.append((".in_proj_qkvgfab", ".g_proj", 3))
|
||||
if getattr(self.config, "q_lora_rank", None) is not None:
|
||||
stacked_params_mapping += [
|
||||
(".fused_qkv_a_proj", ".q_a_proj", 0),
|
||||
(".fused_qkv_a_proj", ".kv_a_proj_with_mqa", 1),
|
||||
]
|
||||
|
||||
use_mega_moe = any(
|
||||
module.use_mega_moe
|
||||
for module in self.modules()
|
||||
if isinstance(module, KimiMoE)
|
||||
)
|
||||
if self.config.is_moe and use_mega_moe:
|
||||
expert_params_mapping = make_kimi_k3_mega_moe_expert_params_mapping(
|
||||
self.config.num_experts
|
||||
)
|
||||
elif self.config.is_moe:
|
||||
expert_params_mapping = fused_moe_make_expert_params_mapping(
|
||||
self,
|
||||
ckpt_gate_proj_name="w1",
|
||||
ckpt_down_proj_name="w2",
|
||||
ckpt_up_proj_name="w3",
|
||||
num_experts=self.config.num_experts,
|
||||
)
|
||||
else:
|
||||
expert_params_mapping = []
|
||||
|
||||
pp_missing_layer_names = get_pp_missing_layer_names(self)
|
||||
params_dict = dict(self.named_parameters())
|
||||
# Under the MXFP4 quant interface the routed experts register unpacked
|
||||
# params (``w13_weight``), while the compressed-tensors checkpoint names
|
||||
# them ``.weight_packed``. Rebind so the expert mapping resolves; scales
|
||||
# already share the ``.weight_scale`` suffix.
|
||||
experts_unpacked = not use_mega_moe and not any(
|
||||
n.endswith("w13_weight_packed") for n in params_dict
|
||||
)
|
||||
loaded_params: set[str] = set()
|
||||
for name, loaded_weight in weights:
|
||||
if "rotary_emb.inv_freq" in name:
|
||||
continue
|
||||
# The multimodal checkpoint prefixes text weights with
|
||||
# ``language_model.``; strip it so names match this draft model's
|
||||
# parameter paths (``model.layers.{i}.``). Non-text weights
|
||||
# (vision_tower, mm_projector, ...) never match a spec layer below.
|
||||
if name.startswith("language_model."):
|
||||
name = name[len("language_model.") :]
|
||||
if experts_unpacked and name.endswith(".weight_packed"):
|
||||
name = name.replace(".weight_packed", ".weight")
|
||||
spec_layer = get_spec_layer_idx_from_weight_name(self.config, name)
|
||||
if spec_layer is None:
|
||||
continue
|
||||
name = self._rewrite_spec_layer_name(spec_layer, name)
|
||||
|
||||
for param_name, weight_name, shard_id in stacked_params_mapping:
|
||||
if weight_name not in name:
|
||||
continue
|
||||
# Routed experts (``.experts.{i}.w1/w2/w3``) are handled by the
|
||||
# expert mapping below; skip them here. Shared experts
|
||||
# (``.shared_experts.``) use gate/up_proj and fall through.
|
||||
if ".experts." in name:
|
||||
continue
|
||||
name_mapped = name.replace(weight_name, param_name)
|
||||
# Only take this mapping if the fused destination actually
|
||||
# exists (e.g. QKV fusion is only present when q_lora is used).
|
||||
if name_mapped not in params_dict:
|
||||
continue
|
||||
if name_mapped in pp_missing_layer_names:
|
||||
continue
|
||||
name = name_mapped
|
||||
param = params_dict[name]
|
||||
weight_loader = param.weight_loader
|
||||
weight_loader(param, loaded_weight, shard_id)
|
||||
break
|
||||
else:
|
||||
for (
|
||||
expert_param_name,
|
||||
expert_weight_name,
|
||||
expert_id,
|
||||
expert_shard_id,
|
||||
) in expert_params_mapping:
|
||||
if expert_weight_name not in name:
|
||||
continue
|
||||
name_mapped = name.replace(expert_weight_name, expert_param_name)
|
||||
if name_mapped in pp_missing_layer_names:
|
||||
continue
|
||||
param = params_dict[name_mapped]
|
||||
weight_loader = param.weight_loader
|
||||
weight_loader(
|
||||
param,
|
||||
loaded_weight,
|
||||
name_mapped,
|
||||
shard_id=expert_shard_id,
|
||||
expert_id=expert_id,
|
||||
)
|
||||
name = name_mapped
|
||||
break
|
||||
else:
|
||||
if name.endswith(".bias") and name not in params_dict:
|
||||
continue
|
||||
remapped_name = maybe_remap_kv_scale_name(name, params_dict)
|
||||
if remapped_name is None:
|
||||
continue
|
||||
name = remapped_name
|
||||
|
||||
# The embedding is shared across MTP layers; only the first
|
||||
# spec layer carries the hoisted (non-".layers") copy.
|
||||
if spec_layer != self.model.mtp_start_layer_idx and (
|
||||
".layers" not in name
|
||||
):
|
||||
continue
|
||||
if name in pp_missing_layer_names:
|
||||
continue
|
||||
# The base model uses an attn-residual scheme whose per-layer
|
||||
# weights (self_attention_res_*, mlp_res_*) are not used by
|
||||
# the draft block; such names have no matching parameter and
|
||||
# are safely skipped.
|
||||
if name not in params_dict:
|
||||
continue
|
||||
|
||||
param = params_dict[name]
|
||||
weight_loader = getattr(
|
||||
param, "weight_loader", default_weight_loader
|
||||
)
|
||||
weight_loader(param, loaded_weight)
|
||||
loaded_params.add(name)
|
||||
|
||||
# Validate that weights were loaded for each expected MTP layer.
|
||||
loaded_layers: set[int] = set()
|
||||
for param_name in loaded_params:
|
||||
spec_layer = get_spec_layer_idx_from_weight_name(self.config, param_name)
|
||||
if spec_layer is not None:
|
||||
loaded_layers.add(spec_layer)
|
||||
for layer_idx in range(
|
||||
self.model.mtp_start_layer_idx,
|
||||
self.model.mtp_start_layer_idx + self.model.num_mtp_layers,
|
||||
):
|
||||
if layer_idx not in loaded_layers:
|
||||
raise ValueError(
|
||||
f"MTP speculative decoding layer {layer_idx} weights "
|
||||
f"missing from checkpoint. The checkpoint may not include "
|
||||
f"the MTP layer weights. Use a checkpoint that includes "
|
||||
f"MTP layer weights, or disable speculative decoding."
|
||||
)
|
||||
|
||||
if use_mega_moe:
|
||||
for module in self.modules():
|
||||
if isinstance(module, KimiMoE) and module.use_mega_moe:
|
||||
module.experts.finalize_weights()
|
||||
|
||||
return loaded_params
|
||||
|
||||
def _rewrite_spec_layer_name(self, spec_layer: int, name: str) -> str:
|
||||
"""Rewrite a checkpoint weight name to this module's parameter path.
|
||||
|
||||
Top-level MTP submodules (enorm/hnorm/eh_proj/shared_head) stay under
|
||||
``model.layers.{spec_layer}.*``; the shared ``embed_tokens`` is hoisted
|
||||
to ``model.*``; everything else is a transformer-block weight and gets
|
||||
``.mtp_block`` inserted.
|
||||
"""
|
||||
spec_layer_weight_names = [
|
||||
"embed_tokens",
|
||||
"enorm",
|
||||
"hnorm",
|
||||
"eh_proj",
|
||||
"shared_head",
|
||||
]
|
||||
shared_weight_names = ["embed_tokens"]
|
||||
spec_layer_weight = False
|
||||
shared_weight = False
|
||||
for weight_name in spec_layer_weight_names:
|
||||
if weight_name in name:
|
||||
spec_layer_weight = True
|
||||
if weight_name in shared_weight_names:
|
||||
shared_weight = True
|
||||
break
|
||||
if not spec_layer_weight:
|
||||
name = name.replace(
|
||||
f"model.layers.{spec_layer}.",
|
||||
f"model.layers.{spec_layer}.mtp_block.",
|
||||
)
|
||||
elif shared_weight:
|
||||
name = name.replace(f"model.layers.{spec_layer}.", "model.")
|
||||
return name
|
||||
@@ -0,0 +1,2 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
@@ -0,0 +1,14 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""CuTe DSL kernels for KimiK3LatentMoETailOp."""
|
||||
|
||||
from .allreduce_rmsnorm_reduce_scatter_early_exit import CollectiveKernel
|
||||
from .fused_add_multicast_gemm import AdaptiveUpProjectionKernel
|
||||
from .lamport_copy import LamportCopyKernel
|
||||
|
||||
__all__ = [
|
||||
"AdaptiveUpProjectionKernel",
|
||||
"CollectiveKernel",
|
||||
"LamportCopyKernel",
|
||||
]
|
||||
+1012
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
+438
@@ -0,0 +1,438 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""Static-M SIMT skinny GEMM with a shared-add multicast epilogue."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
||||
import cuda.bindings.driver as cuda
|
||||
import cutlass
|
||||
import cutlass.cute as cute
|
||||
import torch
|
||||
from cutlass import BFloat16, Float32, Int64, const_expr
|
||||
from cutlass.cute.runtime import from_dlpack
|
||||
|
||||
from .primitives import (
|
||||
CUDAGraphCompatibleWrapper,
|
||||
bf16x2_to_u32,
|
||||
bf16x4_to_packed_u32x2,
|
||||
bf16x8_to_packed_u32x4,
|
||||
sanitize_negative_zero,
|
||||
sanitize_negative_zero_u32,
|
||||
sanitize_negative_zero_u32x2,
|
||||
store_global_u32,
|
||||
store_global_u32x2,
|
||||
store_global_u32x4,
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class SkinnyConfig:
|
||||
block_size: int = 224
|
||||
outputs_per_block: int = 2
|
||||
k_unroll: int = 2
|
||||
vector_width: int = 8
|
||||
prefetch_b_before_pdl: bool = True
|
||||
|
||||
|
||||
def config_for_m(num_rows: int, shard_dim: int = 896) -> SkinnyConfig:
|
||||
if shard_dim == 448:
|
||||
if num_rows >= 6:
|
||||
return SkinnyConfig(
|
||||
block_size=224,
|
||||
outputs_per_block=2,
|
||||
k_unroll=1,
|
||||
)
|
||||
outputs_per_block = 2 if num_rows <= 3 else 4
|
||||
return SkinnyConfig(
|
||||
block_size=448,
|
||||
outputs_per_block=outputs_per_block,
|
||||
k_unroll=1,
|
||||
)
|
||||
if num_rows == 1:
|
||||
return SkinnyConfig(outputs_per_block=8, k_unroll=1)
|
||||
return SkinnyConfig(outputs_per_block=4)
|
||||
|
||||
|
||||
def _as_cute(tensor: torch.Tensor):
|
||||
return from_dlpack(
|
||||
CUDAGraphCompatibleWrapper(tensor.detach()),
|
||||
assumed_align=16,
|
||||
)
|
||||
|
||||
|
||||
class FusedAddMulticastSkinnyGemm:
|
||||
"""SIMT GEMM adapted from the existing Skinny GEMM."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
num_rows: int,
|
||||
hidden_dim: int,
|
||||
config: SkinnyConfig,
|
||||
) -> None:
|
||||
if config.block_size % 32:
|
||||
raise ValueError("skinny block_size must be a multiple of 32")
|
||||
self.num_rows = num_rows
|
||||
self.hidden_dim = hidden_dim
|
||||
self.block_size = config.block_size
|
||||
self.outputs_per_block = config.outputs_per_block
|
||||
self.k_unroll = config.k_unroll
|
||||
self.vector_width = config.vector_width
|
||||
self.prefetch_b_before_pdl = config.prefetch_b_before_pdl
|
||||
self.num_warps = config.block_size // 32
|
||||
|
||||
@cute.jit
|
||||
def __call__(
|
||||
self,
|
||||
gA: cute.Tensor,
|
||||
gB: cute.Tensor,
|
||||
gShared: cute.Tensor,
|
||||
output_multicast_ptr: Int64,
|
||||
stream: cuda.CUstream,
|
||||
) -> None:
|
||||
n = cute.size(gB, mode=[0])
|
||||
k = cute.size(gA, mode=[1])
|
||||
copy_a = cute.make_copy_atom(
|
||||
cute.nvgpu.CopyG2ROp(),
|
||||
BFloat16,
|
||||
num_bits_per_copy=self.vector_width * BFloat16.width,
|
||||
load_cache_mode=cute.nvgpu.LoadCacheMode.ALWAYS,
|
||||
)
|
||||
copy_b = cute.make_copy_atom(
|
||||
cute.nvgpu.CopyG2ROp(),
|
||||
BFloat16,
|
||||
num_bits_per_copy=self.vector_width * BFloat16.width,
|
||||
load_cache_mode=cute.nvgpu.LoadCacheMode.STREAMING,
|
||||
)
|
||||
self.kernel(
|
||||
gA,
|
||||
gB,
|
||||
gShared,
|
||||
output_multicast_ptr,
|
||||
k,
|
||||
copy_a,
|
||||
copy_b,
|
||||
).launch(
|
||||
grid=[cute.ceil_div(n, self.outputs_per_block), 1, 1],
|
||||
block=[self.block_size, 1, 1],
|
||||
smem=(self.num_rows * self.outputs_per_block * self.num_warps * 4),
|
||||
stream=stream,
|
||||
use_pdl=True,
|
||||
min_blocks_per_mp=1,
|
||||
)
|
||||
|
||||
@cute.kernel
|
||||
def kernel(
|
||||
self,
|
||||
gA: cute.Tensor,
|
||||
gB: cute.Tensor,
|
||||
gShared: cute.Tensor,
|
||||
output_multicast_ptr: Int64,
|
||||
k_extent: cutlass.Int32,
|
||||
copy_a: cute.CopyAtom,
|
||||
copy_b: cute.CopyAtom,
|
||||
) -> None:
|
||||
tidx, _, _ = cute.arch.thread_idx()
|
||||
block_idx, _, _ = cute.arch.block_idx()
|
||||
warp_idx = cute.arch.warp_idx()
|
||||
|
||||
outputs_per_block: cutlass.Constexpr = self.outputs_per_block
|
||||
vector_width: cutlass.Constexpr = self.vector_width
|
||||
block_size: cutlass.Constexpr = self.block_size
|
||||
num_warps: cutlass.Constexpr = self.num_warps
|
||||
num_rows: cutlass.Constexpr = self.num_rows
|
||||
|
||||
acc = cute.make_rmem_tensor(
|
||||
cute.make_layout(
|
||||
(num_rows, outputs_per_block),
|
||||
stride=(outputs_per_block, 1),
|
||||
),
|
||||
Float32,
|
||||
)
|
||||
acc.fill(0.0)
|
||||
|
||||
n_base = block_idx * outputs_per_block
|
||||
k_tile_size: cutlass.Constexpr = block_size * vector_width
|
||||
num_k_tiles = k_extent // k_tile_size
|
||||
gA_vec = cute.logical_divide(gA, (None, vector_width))
|
||||
gB_vec = cute.logical_divide(gB, (None, vector_width))
|
||||
tA_all = cute.logical_divide(gA_vec, (None, (None, block_size)))
|
||||
tB_all = cute.logical_divide(gB_vec, (None, (None, block_size)))
|
||||
tA = tA_all[None, (None, (tidx, None))]
|
||||
|
||||
a_regs = cute.make_rmem_tensor(
|
||||
cute.make_layout(
|
||||
(num_rows, vector_width),
|
||||
stride=(vector_width, 1),
|
||||
),
|
||||
BFloat16,
|
||||
)
|
||||
b_regs = cute.make_rmem_tensor(
|
||||
cute.make_layout(
|
||||
(outputs_per_block, vector_width),
|
||||
stride=(vector_width, 1),
|
||||
),
|
||||
BFloat16,
|
||||
)
|
||||
|
||||
if const_expr(self.prefetch_b_before_pdl):
|
||||
for ni in cutlass.range_constexpr(outputs_per_block):
|
||||
tB = tB_all[n_base + ni, (None, (tidx, None))]
|
||||
cute.copy(copy_b, tB[None, 0], b_regs[ni, None])
|
||||
|
||||
cute.arch.griddepcontrol_wait()
|
||||
|
||||
for mi in cutlass.range_constexpr(num_rows):
|
||||
cute.copy(copy_a, tA[mi, None, 0], a_regs[mi, None])
|
||||
if const_expr(not self.prefetch_b_before_pdl):
|
||||
for ni in cutlass.range_constexpr(outputs_per_block):
|
||||
tB = tB_all[n_base + ni, (None, (tidx, None))]
|
||||
cute.copy(copy_b, tB[None, 0], b_regs[ni, None])
|
||||
for vi in cutlass.range_constexpr(vector_width):
|
||||
for mi in cutlass.range_constexpr(num_rows):
|
||||
for ni in cutlass.range_constexpr(outputs_per_block):
|
||||
acc[mi, ni] = acc[mi, ni] + a_regs[mi, vi].to(Float32) * b_regs[
|
||||
ni, vi
|
||||
].to(Float32)
|
||||
|
||||
for k_tile in cutlass.range(1, num_k_tiles, unroll=self.k_unroll):
|
||||
for mi in cutlass.range_constexpr(num_rows):
|
||||
cute.copy(
|
||||
copy_a,
|
||||
tA[mi, None, k_tile],
|
||||
a_regs[mi, None],
|
||||
)
|
||||
for ni in cutlass.range_constexpr(outputs_per_block):
|
||||
tB = tB_all[n_base + ni, (None, (tidx, None))]
|
||||
cute.copy(copy_b, tB[None, k_tile], b_regs[ni, None])
|
||||
for vi in cutlass.range_constexpr(vector_width):
|
||||
for mi in cutlass.range_constexpr(num_rows):
|
||||
for ni in cutlass.range_constexpr(outputs_per_block):
|
||||
acc[mi, ni] = acc[mi, ni] + a_regs[mi, vi].to(Float32) * b_regs[
|
||||
ni, vi
|
||||
].to(Float32)
|
||||
|
||||
for mi in cutlass.range_constexpr(num_rows):
|
||||
for ni in cutlass.range_constexpr(outputs_per_block):
|
||||
acc[mi, ni] = cute.arch.warp_reduction_sum(acc[mi, ni])
|
||||
|
||||
smem_layout = cute.make_layout(
|
||||
(num_rows, outputs_per_block, num_warps),
|
||||
stride=(outputs_per_block * num_warps, num_warps, 1),
|
||||
)
|
||||
smem = cutlass.utils.SmemAllocator()
|
||||
partials = smem.allocate_tensor(
|
||||
Float32,
|
||||
smem_layout,
|
||||
byte_alignment=16,
|
||||
)
|
||||
with cute.arch.elect_one():
|
||||
for mi in cutlass.range_constexpr(num_rows):
|
||||
for ni in cutlass.range_constexpr(outputs_per_block):
|
||||
partials[mi, ni, warp_idx] = acc[mi, ni]
|
||||
|
||||
cute.arch.sync_threads()
|
||||
if tidx == 0:
|
||||
fused = cute.make_rmem_tensor(
|
||||
cute.make_layout((outputs_per_block,)),
|
||||
BFloat16,
|
||||
)
|
||||
for mi in cutlass.range_constexpr(num_rows):
|
||||
for ni in cutlass.range_constexpr(outputs_per_block):
|
||||
total = (
|
||||
partials[mi, ni, None]
|
||||
.load()
|
||||
.reduce(
|
||||
cute.ReductionOp.ADD,
|
||||
init_val=Float32(0.0),
|
||||
reduction_profile=0,
|
||||
)
|
||||
)
|
||||
gemm_value = Float32(total).to(BFloat16)
|
||||
fused[ni] = (
|
||||
gemm_value.to(Float32) + gShared[mi, n_base + ni].to(Float32)
|
||||
).to(BFloat16)
|
||||
output_offset = Int64((mi * self.hidden_dim + n_base) * 2)
|
||||
if const_expr(outputs_per_block == 2):
|
||||
packed = sanitize_negative_zero_u32(bf16x2_to_u32(fused.load()))
|
||||
store_global_u32(
|
||||
output_multicast_ptr + output_offset,
|
||||
packed,
|
||||
)
|
||||
elif const_expr(outputs_per_block == 4):
|
||||
packed = sanitize_negative_zero_u32x2(
|
||||
bf16x4_to_packed_u32x2(fused.load())
|
||||
)
|
||||
store_global_u32x2(
|
||||
output_multicast_ptr + output_offset,
|
||||
packed,
|
||||
)
|
||||
else:
|
||||
packed = sanitize_negative_zero(
|
||||
bf16x8_to_packed_u32x4(fused.load())
|
||||
)
|
||||
store_global_u32x4(
|
||||
output_multicast_ptr + output_offset,
|
||||
packed,
|
||||
)
|
||||
|
||||
cute.arch.griddepcontrol_launch_dependents()
|
||||
|
||||
|
||||
_COMPILED: dict[tuple[object, ...], object] = {}
|
||||
|
||||
|
||||
def compile_kernel(
|
||||
*,
|
||||
num_rows: int,
|
||||
latent_dim: int,
|
||||
hidden_dim: int,
|
||||
shard_dim: int,
|
||||
config: SkinnyConfig,
|
||||
):
|
||||
key = (
|
||||
torch.accelerator.current_device_index(),
|
||||
num_rows,
|
||||
latent_dim,
|
||||
hidden_dim,
|
||||
shard_dim,
|
||||
config,
|
||||
)
|
||||
if key in _COMPILED:
|
||||
return _COMPILED[key]
|
||||
|
||||
from cutlass.cute.runtime import make_fake_stream, make_fake_tensor
|
||||
|
||||
if shard_dim % config.outputs_per_block:
|
||||
raise ValueError("shard_dim must be divisible by outputs_per_block")
|
||||
if latent_dim % (config.block_size * config.vector_width):
|
||||
raise ValueError("latent_dim must be divisible by block_size * vector_width")
|
||||
|
||||
a = make_fake_tensor(
|
||||
BFloat16,
|
||||
(num_rows, latent_dim),
|
||||
stride=(latent_dim, 1),
|
||||
assumed_align=16,
|
||||
)
|
||||
b = make_fake_tensor(
|
||||
BFloat16,
|
||||
(shard_dim, latent_dim),
|
||||
stride=(latent_dim, 1),
|
||||
assumed_align=16,
|
||||
)
|
||||
shared = make_fake_tensor(
|
||||
BFloat16,
|
||||
(num_rows, shard_dim),
|
||||
stride=(hidden_dim, 1),
|
||||
assumed_align=16,
|
||||
)
|
||||
compiled = cute.compile(
|
||||
FusedAddMulticastSkinnyGemm(
|
||||
num_rows=num_rows,
|
||||
hidden_dim=hidden_dim,
|
||||
config=config,
|
||||
),
|
||||
a,
|
||||
b,
|
||||
shared,
|
||||
Int64(0),
|
||||
make_fake_stream(),
|
||||
options="--ptxas-options -maxrregcount=128",
|
||||
)
|
||||
_COMPILED[key] = compiled
|
||||
return compiled
|
||||
|
||||
|
||||
class FusedAddMulticastSkinnyGemmKernel:
|
||||
"""Buffer-free compiled launcher for one static M."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
rank: int,
|
||||
tp_size: int,
|
||||
latent_dim: int,
|
||||
hidden_dim: int,
|
||||
num_rows: int,
|
||||
) -> None:
|
||||
if not 1 <= num_rows <= 8:
|
||||
raise ValueError("skinny backend requires static M in [1, 8]")
|
||||
if hidden_dim % tp_size:
|
||||
raise ValueError("hidden_dim must be divisible by TP size")
|
||||
self.rank = rank
|
||||
self.latent_dim = latent_dim
|
||||
self.hidden_dim = hidden_dim
|
||||
self.shard_dim = hidden_dim // tp_size
|
||||
self.num_rows = num_rows
|
||||
self._skinny = compile_kernel(
|
||||
num_rows=num_rows,
|
||||
latent_dim=latent_dim,
|
||||
hidden_dim=hidden_dim,
|
||||
shard_dim=self.shard_dim,
|
||||
config=config_for_m(num_rows, self.shard_dim),
|
||||
)
|
||||
|
||||
def __call__(
|
||||
self,
|
||||
latent: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
shared_shard: torch.Tensor,
|
||||
mailbox: torch.Tensor,
|
||||
mailbox_multicast_ptr: int,
|
||||
) -> torch.Tensor:
|
||||
device = mailbox.device
|
||||
expected = (
|
||||
(
|
||||
latent,
|
||||
(self.num_rows, self.latent_dim),
|
||||
torch.bfloat16,
|
||||
"latent",
|
||||
),
|
||||
(
|
||||
weight,
|
||||
(self.shard_dim, self.latent_dim),
|
||||
torch.bfloat16,
|
||||
"weight",
|
||||
),
|
||||
(
|
||||
shared_shard,
|
||||
(mailbox.shape[1], self.shard_dim),
|
||||
torch.bfloat16,
|
||||
"shared_shard",
|
||||
),
|
||||
(
|
||||
mailbox,
|
||||
(1, mailbox.shape[1], self.hidden_dim),
|
||||
torch.bfloat16,
|
||||
"mailbox",
|
||||
),
|
||||
)
|
||||
for tensor, shape, dtype, name in expected:
|
||||
if (
|
||||
tensor.shape != shape
|
||||
or tensor.dtype != dtype
|
||||
or tensor.device != device
|
||||
):
|
||||
raise ValueError(f"{name} must be CUDA {dtype} {list(shape)}")
|
||||
if (
|
||||
not latent.is_contiguous()
|
||||
or not weight.is_contiguous()
|
||||
or shared_shard.stride() != (self.hidden_dim, 1)
|
||||
or not mailbox.is_contiguous()
|
||||
):
|
||||
raise ValueError("skinny up-projection inputs have unsupported strides")
|
||||
if mailbox.shape[1] < self.num_rows:
|
||||
raise ValueError("mailbox capacity is smaller than runtime M")
|
||||
|
||||
with torch.accelerator.device_index(device.index):
|
||||
self._skinny(
|
||||
_as_cute(latent),
|
||||
_as_cute(weight),
|
||||
_as_cute(shared_shard[: self.num_rows]),
|
||||
Int64(mailbox_multicast_ptr + self.rank * self.shard_dim * 2),
|
||||
cuda.CUstream(torch.cuda.current_stream(device).cuda_stream),
|
||||
)
|
||||
return mailbox
|
||||
@@ -0,0 +1,226 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""Lamport mailbox-to-local copy and reset."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import functools
|
||||
|
||||
import cuda.bindings.driver as cuda
|
||||
import cutlass
|
||||
import cutlass.cute as cute
|
||||
import torch
|
||||
from cutlass.cute.runtime import make_fake_compact_tensor, make_fake_stream
|
||||
|
||||
from .primitives import (
|
||||
VEC_BF16,
|
||||
fragment_is_dirty,
|
||||
load_global_u32x4,
|
||||
store_global_u32x4,
|
||||
store_lamport_sentinel_128,
|
||||
to_cute,
|
||||
to_cute_dynamic_m,
|
||||
)
|
||||
|
||||
|
||||
class LamportCopy:
|
||||
"""Consume the local physical copy of an NVLS-multicast mailbox."""
|
||||
|
||||
def __init__(self, hidden_dim: int, ctas: int, threads: int):
|
||||
self.hidden_dim = hidden_dim
|
||||
self.ctas = ctas
|
||||
self.threads = threads
|
||||
|
||||
@cute.jit
|
||||
def __call__(
|
||||
self,
|
||||
symmetric_mailbox: cute.Tensor,
|
||||
local_output: cute.Tensor,
|
||||
m: cutlass.Int32,
|
||||
stream: cuda.CUstream,
|
||||
):
|
||||
self.kernel(symmetric_mailbox, local_output, m).launch(
|
||||
grid=(self.ctas, 1, 1),
|
||||
block=(self.threads, 1, 1),
|
||||
stream=stream,
|
||||
use_pdl=True,
|
||||
)
|
||||
|
||||
@cute.kernel
|
||||
def kernel(
|
||||
self,
|
||||
symmetric_mailbox: cute.Tensor,
|
||||
local_output: cute.Tensor,
|
||||
m: cutlass.Int32,
|
||||
):
|
||||
# The CTA may be scheduled early, but mailbox inspection must not pass
|
||||
# the producer GEMM's programmatic completion point.
|
||||
cute.arch.griddepcontrol_wait()
|
||||
|
||||
tidx, _, _ = cute.arch.thread_idx()
|
||||
block, _, _ = cute.arch.block_idx()
|
||||
thread = cutlass.Int64(block * self.threads + tidx)
|
||||
stride = cutlass.Int64(self.ctas * self.threads)
|
||||
fragments = cutlass.Int64(m) * cutlass.Int64(self.hidden_dim // VEC_BF16)
|
||||
|
||||
fragment = thread
|
||||
while fragment < fragments:
|
||||
element = fragment * VEC_BF16
|
||||
source = cute.make_ptr(
|
||||
cutlass.BFloat16,
|
||||
(symmetric_mailbox.iterator + element).llvm_ptr,
|
||||
cute.AddressSpace.gmem,
|
||||
assumed_align=16,
|
||||
)
|
||||
packed = load_global_u32x4(source, volatile=True)
|
||||
while fragment_is_dirty(packed):
|
||||
packed = load_global_u32x4(source, volatile=True)
|
||||
|
||||
destination = cutlass.Int64((local_output.iterator + element).toint())
|
||||
store_global_u32x4(destination, packed, volatile=False)
|
||||
fragment = fragment + stride
|
||||
|
||||
# The returned ordinary tensor is complete. A same-stream successor
|
||||
# may overlap the mailbox cleanup below.
|
||||
cute.arch.griddepcontrol_launch_dependents()
|
||||
|
||||
fragment = thread
|
||||
while fragment < fragments:
|
||||
element = fragment * VEC_BF16
|
||||
source = cute.make_ptr(
|
||||
cutlass.BFloat16,
|
||||
(symmetric_mailbox.iterator + element).llvm_ptr,
|
||||
cute.AddressSpace.gmem,
|
||||
assumed_align=16,
|
||||
)
|
||||
store_lamport_sentinel_128(source)
|
||||
fragment = fragment + stride
|
||||
|
||||
|
||||
@functools.cache
|
||||
def compile_kernel(
|
||||
hidden_dim: int,
|
||||
max_m: int,
|
||||
ctas: int,
|
||||
threads: int,
|
||||
device_index: int,
|
||||
):
|
||||
if hidden_dim <= 0 or hidden_dim % VEC_BF16:
|
||||
raise ValueError("hidden_dim must be a positive multiple of 8")
|
||||
if max_m <= 0:
|
||||
raise ValueError("max_m must be positive")
|
||||
if ctas <= 0:
|
||||
raise ValueError("Lamport copy ctas must be positive")
|
||||
if not 1 <= threads <= 1024:
|
||||
raise ValueError("Lamport copy threads must be in [1, 1024]")
|
||||
with torch.accelerator.device_index(device_index):
|
||||
mailbox = make_fake_compact_tensor(
|
||||
cutlass.BFloat16, (max_m * hidden_dim,), assumed_align=16
|
||||
)
|
||||
output = make_fake_compact_tensor(
|
||||
cutlass.BFloat16,
|
||||
(cute.sym_int32(divisibility=VEC_BF16),),
|
||||
assumed_align=16,
|
||||
)
|
||||
return cute.compile(
|
||||
LamportCopy(hidden_dim, ctas, threads),
|
||||
mailbox,
|
||||
output,
|
||||
cutlass.Int32(max_m),
|
||||
make_fake_stream(),
|
||||
)
|
||||
|
||||
|
||||
def launch(
|
||||
symmetric_mailbox: torch.Tensor,
|
||||
local_output: torch.Tensor,
|
||||
*,
|
||||
m: int,
|
||||
hidden_dim: int,
|
||||
max_m: int,
|
||||
ctas: int,
|
||||
threads: int,
|
||||
) -> None:
|
||||
if not 1 <= m <= max_m:
|
||||
raise ValueError(f"runtime M={m} must be in [1, {max_m}]")
|
||||
if (
|
||||
symmetric_mailbox.ndim != 3
|
||||
or symmetric_mailbox.dtype != torch.bfloat16
|
||||
or not symmetric_mailbox.is_cuda
|
||||
or not symmetric_mailbox.is_contiguous()
|
||||
or symmetric_mailbox.shape[0] != 1
|
||||
or symmetric_mailbox.shape[2] != hidden_dim
|
||||
):
|
||||
raise ValueError(
|
||||
f"symmetric_mailbox must be contiguous CUDA BF16 [1,M,{hidden_dim}]"
|
||||
)
|
||||
if symmetric_mailbox.shape[1] != max_m:
|
||||
raise ValueError("symmetric mailbox must retain its full max_m capacity")
|
||||
if (
|
||||
local_output.shape != (1, m, hidden_dim)
|
||||
or local_output.dtype != torch.bfloat16
|
||||
or local_output.device != symmetric_mailbox.device
|
||||
or not local_output.is_contiguous()
|
||||
):
|
||||
raise ValueError("local_output must be contiguous CUDA BF16 [1,M,H]")
|
||||
|
||||
device_index = symmetric_mailbox.device.index
|
||||
stream = cuda.CUstream(
|
||||
torch.cuda.current_stream(symmetric_mailbox.device).cuda_stream
|
||||
)
|
||||
compile_kernel(hidden_dim, max_m, ctas, threads, device_index)(
|
||||
to_cute(symmetric_mailbox.flatten(), 16),
|
||||
to_cute_dynamic_m(
|
||||
local_output.flatten(),
|
||||
mode=0,
|
||||
assumed_align=16,
|
||||
),
|
||||
cutlass.Int32(m),
|
||||
stream,
|
||||
)
|
||||
|
||||
|
||||
class LamportCopyKernel:
|
||||
"""Copy a borrowed symmetric mailbox into a fresh local tensor."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
hidden_dim: int,
|
||||
max_m: int,
|
||||
ctas: int,
|
||||
threads: int,
|
||||
) -> None:
|
||||
self.hidden_dim = hidden_dim
|
||||
self.max_m = max_m
|
||||
self.ctas = ctas
|
||||
self.threads = threads
|
||||
compile_kernel(
|
||||
hidden_dim,
|
||||
max_m,
|
||||
ctas,
|
||||
threads,
|
||||
torch.accelerator.current_device_index(),
|
||||
)
|
||||
|
||||
def __call__(self, symmetric_mailbox: torch.Tensor, *, m: int) -> torch.Tensor:
|
||||
if not symmetric_mailbox.is_cuda:
|
||||
raise ValueError("symmetric_mailbox must be a CUDA tensor")
|
||||
device = symmetric_mailbox.device
|
||||
with torch.accelerator.device_index(device.index):
|
||||
output = torch.empty(
|
||||
(1, m, self.hidden_dim),
|
||||
dtype=torch.bfloat16,
|
||||
device=device,
|
||||
)
|
||||
launch(
|
||||
symmetric_mailbox,
|
||||
output,
|
||||
m=m,
|
||||
hidden_dim=self.hidden_dim,
|
||||
max_m=self.max_m,
|
||||
ctas=self.ctas,
|
||||
threads=self.threads,
|
||||
)
|
||||
return output
|
||||
@@ -0,0 +1,437 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""Shared CuTe DSL primitives; this module does not define a CUDA kernel."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import cutlass
|
||||
import cutlass.cute as cute
|
||||
import torch
|
||||
from cutlass import BFloat16, Float32, Int32, Int64, Uint16, Uint32
|
||||
from cutlass._mlir import ir
|
||||
from cutlass._mlir.dialects import llvm, vector
|
||||
from cutlass.cute.runtime import from_dlpack
|
||||
from cutlass.cutlass_dsl import T, dsl_user_op
|
||||
|
||||
VEC_BF16 = 8
|
||||
PACKED_BYTES = 16
|
||||
NUM_LAMPORT_BUFFERS = 3
|
||||
NEG_ZERO_F32_BITS = 0x80000000
|
||||
NEG_ZERO_BF16_BITS = 0x8000
|
||||
|
||||
|
||||
class CUDAGraphCompatibleWrapper:
|
||||
"""DLPack view that does not synchronize with the producer stream."""
|
||||
|
||||
def __init__(self, tensor: torch.Tensor):
|
||||
self.tensor = tensor
|
||||
|
||||
def __dlpack__(self, stream=None):
|
||||
return self.tensor.__dlpack__(stream=-1)
|
||||
|
||||
def __dlpack_device__(self):
|
||||
return self.tensor.__dlpack_device__()
|
||||
|
||||
|
||||
def to_cute(tensor: torch.Tensor, assumed_align: int = 16) -> cute.Tensor:
|
||||
return from_dlpack(
|
||||
CUDAGraphCompatibleWrapper(tensor.detach()), assumed_align=assumed_align
|
||||
)
|
||||
|
||||
|
||||
def to_cute_dynamic_m(
|
||||
tensor: torch.Tensor,
|
||||
*,
|
||||
mode: int,
|
||||
assumed_align: int = 16,
|
||||
) -> cute.Tensor:
|
||||
"""Expose exactly one compact tensor mode as a runtime shape.
|
||||
|
||||
Model dimensions remain part of the compiled tensor type. Only the token
|
||||
mode is symbolic, so changing M within an Op's capacity reuses the same
|
||||
compiled kernel.
|
||||
"""
|
||||
|
||||
return to_cute(tensor, assumed_align).mark_compact_shape_dynamic(
|
||||
mode=mode,
|
||||
stride_order=tensor.dim_order(),
|
||||
)
|
||||
|
||||
|
||||
@dsl_user_op
|
||||
def load_global_u32x4(
|
||||
pointer: cute.Pointer,
|
||||
*,
|
||||
volatile: cutlass.Constexpr[bool] = False,
|
||||
loc=None,
|
||||
ip=None,
|
||||
):
|
||||
"""Load one 128-bit fragment as four u32 registers.
|
||||
|
||||
The volatile form is the Lamport polling load. Marking the asm
|
||||
side-effecting prevents loop-invariant motion and common-subexpression
|
||||
elimination across polling iterations.
|
||||
"""
|
||||
|
||||
address = pointer.toint(loc=loc, ip=ip)
|
||||
opcode = "ld.volatile.global.v4.u32" if volatile else "ld.global.v4.u32"
|
||||
out = llvm.inline_asm(
|
||||
llvm.StructType.get_literal([T.i32()] * 4),
|
||||
[address.ir_value(loc=loc, ip=ip)],
|
||||
f"{opcode} {{$0, $1, $2, $3}}, [$4];",
|
||||
"=r,=r,=r,=r,l",
|
||||
has_side_effects=volatile,
|
||||
is_align_stack=False,
|
||||
asm_dialect=llvm.AsmDialect.AD_ATT,
|
||||
loc=loc,
|
||||
ip=ip,
|
||||
)
|
||||
packed = vector.from_elements(
|
||||
ir.VectorType.get([4], T.i32(), loc=loc),
|
||||
[llvm.extractvalue(T.i32(), out, [i], loc=loc, ip=ip) for i in range(4)],
|
||||
loc=loc,
|
||||
ip=ip,
|
||||
)
|
||||
return cute.TensorSSA(packed, 4, Uint32)
|
||||
|
||||
|
||||
@dsl_user_op
|
||||
def store_global_u32x4(
|
||||
address: Int64,
|
||||
packed,
|
||||
*,
|
||||
volatile: cutlass.Constexpr[bool] = False,
|
||||
loc=None,
|
||||
ip=None,
|
||||
) -> None:
|
||||
"""Store four packed words to an ordinary or NVLS multicast global VA."""
|
||||
|
||||
words = [packed[i].ir_value(loc=loc, ip=ip) for i in range(4)]
|
||||
opcode = "st.volatile.global.v4.u32" if volatile else "st.global.v4.u32"
|
||||
llvm.inline_asm(
|
||||
None,
|
||||
[address.ir_value(loc=loc, ip=ip), *words],
|
||||
f"{opcode} [$0], {{$1, $2, $3, $4}};",
|
||||
"l,r,r,r,r",
|
||||
has_side_effects=True,
|
||||
is_align_stack=False,
|
||||
asm_dialect=llvm.AsmDialect.AD_ATT,
|
||||
loc=loc,
|
||||
ip=ip,
|
||||
)
|
||||
|
||||
|
||||
@dsl_user_op
|
||||
def store_global_u32x2(
|
||||
address: Int64,
|
||||
packed,
|
||||
*,
|
||||
loc=None,
|
||||
ip=None,
|
||||
) -> None:
|
||||
words = [packed[i].ir_value(loc=loc, ip=ip) for i in range(2)]
|
||||
llvm.inline_asm(
|
||||
None,
|
||||
[address.ir_value(loc=loc, ip=ip), *words],
|
||||
"st.global.v2.u32 [$0], {$1, $2};",
|
||||
"l,r,r",
|
||||
has_side_effects=True,
|
||||
is_align_stack=False,
|
||||
asm_dialect=llvm.AsmDialect.AD_ATT,
|
||||
loc=loc,
|
||||
ip=ip,
|
||||
)
|
||||
|
||||
|
||||
@dsl_user_op
|
||||
def store_global_u32(
|
||||
address: Int64,
|
||||
word: Uint32,
|
||||
*,
|
||||
loc=None,
|
||||
ip=None,
|
||||
) -> None:
|
||||
llvm.inline_asm(
|
||||
None,
|
||||
[
|
||||
address.ir_value(loc=loc, ip=ip),
|
||||
word.ir_value(loc=loc, ip=ip),
|
||||
],
|
||||
"st.global.u32 [$0], $1;",
|
||||
"l,r",
|
||||
has_side_effects=True,
|
||||
is_align_stack=False,
|
||||
asm_dialect=llvm.AsmDialect.AD_ATT,
|
||||
loc=loc,
|
||||
ip=ip,
|
||||
)
|
||||
|
||||
|
||||
@dsl_user_op
|
||||
def store_lamport_sentinel_128(pointer: cute.Pointer, *, loc=None, ip=None) -> None:
|
||||
"""Reset one Lamport fragment to four FP32 negative-zero bit patterns."""
|
||||
|
||||
address = pointer.toint(loc=loc, ip=ip)
|
||||
value = Uint32(NEG_ZERO_F32_BITS).ir_value(loc=loc, ip=ip)
|
||||
llvm.inline_asm(
|
||||
None,
|
||||
[address.ir_value(loc=loc, ip=ip), value, value, value, value],
|
||||
"st.global.v4.u32 [$0], {$1, $2, $3, $4};",
|
||||
"l,r,r,r,r",
|
||||
has_side_effects=True,
|
||||
is_align_stack=False,
|
||||
asm_dialect=llvm.AsmDialect.AD_ATT,
|
||||
loc=loc,
|
||||
ip=ip,
|
||||
)
|
||||
|
||||
|
||||
@dsl_user_op
|
||||
def red_async_release_gpu_add_u32(
|
||||
pointer: cute.Pointer, value: Uint32, *, loc=None, ip=None
|
||||
) -> None:
|
||||
"""The exact SM100 arrival primitive used by upstream LamportFlags."""
|
||||
|
||||
address = pointer.toint(loc=loc, ip=ip)
|
||||
llvm.inline_asm(
|
||||
None,
|
||||
[
|
||||
address.ir_value(loc=loc, ip=ip),
|
||||
value.ir_value(loc=loc, ip=ip),
|
||||
],
|
||||
"red.async.release.global.gpu.add.u32 [$0], $1;",
|
||||
"l,r",
|
||||
has_side_effects=True,
|
||||
is_align_stack=False,
|
||||
asm_dialect=llvm.AsmDialect.AD_ATT,
|
||||
loc=loc,
|
||||
ip=ip,
|
||||
)
|
||||
|
||||
|
||||
@dsl_user_op
|
||||
def load_volatile_u32(pointer: cute.Pointer, *, loc=None, ip=None) -> Uint32:
|
||||
address = pointer.toint(loc=loc, ip=ip)
|
||||
return Uint32(
|
||||
llvm.inline_asm(
|
||||
T.i32(),
|
||||
[address.ir_value(loc=loc, ip=ip)],
|
||||
"ld.volatile.global.u32 $0, [$1];",
|
||||
"=r,l",
|
||||
has_side_effects=True,
|
||||
is_align_stack=False,
|
||||
asm_dialect=llvm.AsmDialect.AD_ATT,
|
||||
loc=loc,
|
||||
ip=ip,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@dsl_user_op
|
||||
def map_shared_to_peer(
|
||||
smem_ptr: cute.Pointer,
|
||||
peer_rank: Int32,
|
||||
*,
|
||||
loc=None,
|
||||
ip=None,
|
||||
) -> Int32:
|
||||
"""Map a local shared-memory slot to the same slot in a peer CTA."""
|
||||
|
||||
smem_address = smem_ptr.toint(loc=loc, ip=ip).ir_value()
|
||||
return Int32(
|
||||
llvm.inline_asm(
|
||||
T.i32(),
|
||||
[smem_address, peer_rank.ir_value(loc=loc, ip=ip)],
|
||||
"mapa.shared::cluster.u32 $0, $1, $2;",
|
||||
"=r,r,r",
|
||||
has_side_effects=False,
|
||||
is_align_stack=False,
|
||||
asm_dialect=llvm.AsmDialect.AD_ATT,
|
||||
loc=loc,
|
||||
ip=ip,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@dsl_user_op
|
||||
def store_shared_cluster_f32(
|
||||
remote_address: Int32,
|
||||
value: Float32,
|
||||
*,
|
||||
loc=None,
|
||||
ip=None,
|
||||
) -> None:
|
||||
llvm.inline_asm(
|
||||
None,
|
||||
[
|
||||
remote_address.ir_value(loc=loc, ip=ip),
|
||||
value.ir_value(loc=loc, ip=ip),
|
||||
],
|
||||
"st.shared::cluster.f32 [$0], $1;",
|
||||
"r,f",
|
||||
has_side_effects=True,
|
||||
is_align_stack=False,
|
||||
asm_dialect=llvm.AsmDialect.AD_ATT,
|
||||
loc=loc,
|
||||
ip=ip,
|
||||
)
|
||||
|
||||
|
||||
@dsl_user_op
|
||||
def packed_u32x4_to_bf16x8(packed, *, loc=None, ip=None):
|
||||
target_type = ir.VectorType.get([VEC_BF16], BFloat16.mlir_type, loc=loc)
|
||||
values = llvm.bitcast(
|
||||
target_type,
|
||||
packed.ir_value(loc=loc, ip=ip),
|
||||
loc=loc,
|
||||
ip=ip,
|
||||
)
|
||||
return cute.TensorSSA(values, VEC_BF16, BFloat16)
|
||||
|
||||
|
||||
@dsl_user_op
|
||||
def bf16x8_to_packed_u32x4(values, *, loc=None, ip=None):
|
||||
target_type = ir.VectorType.get([4], T.i32(), loc=loc)
|
||||
packed = llvm.bitcast(
|
||||
target_type,
|
||||
values.ir_value(loc=loc, ip=ip),
|
||||
loc=loc,
|
||||
ip=ip,
|
||||
)
|
||||
return cute.TensorSSA(packed, 4, Uint32)
|
||||
|
||||
|
||||
@dsl_user_op
|
||||
def bf16x4_to_packed_u32x2(values, *, loc=None, ip=None):
|
||||
target_type = ir.VectorType.get([2], T.i32(), loc=loc)
|
||||
packed = llvm.bitcast(
|
||||
target_type,
|
||||
values.ir_value(loc=loc, ip=ip),
|
||||
loc=loc,
|
||||
ip=ip,
|
||||
)
|
||||
return cute.TensorSSA(packed, 2, Uint32)
|
||||
|
||||
|
||||
@dsl_user_op
|
||||
def bf16x2_to_u32(values, *, loc=None, ip=None):
|
||||
packed = llvm.bitcast(
|
||||
T.i32(),
|
||||
values.ir_value(loc=loc, ip=ip),
|
||||
loc=loc,
|
||||
ip=ip,
|
||||
)
|
||||
return Uint32(packed)
|
||||
|
||||
|
||||
@cute.jit
|
||||
def sanitize_negative_zero_u32(word):
|
||||
low = Uint16(word & Uint32(0xFFFF))
|
||||
high = Uint16(word >> Uint32(16))
|
||||
if low == Uint16(NEG_ZERO_BF16_BITS):
|
||||
word = word & Uint32(0xFFFF0000)
|
||||
if high == Uint16(NEG_ZERO_BF16_BITS):
|
||||
word = word & Uint32(0x0000FFFF)
|
||||
return word
|
||||
|
||||
|
||||
@cute.jit
|
||||
def sanitize_negative_zero_u32x2(packed):
|
||||
result = cute.make_rmem_tensor(cute.make_layout((2,)), Uint32)
|
||||
for i in cutlass.range_constexpr(2):
|
||||
result[i] = sanitize_negative_zero_u32(packed[i])
|
||||
return result.load()
|
||||
|
||||
|
||||
@cute.jit
|
||||
def sanitize_negative_zero(packed):
|
||||
"""Turn real BF16 -0 into +0 so it cannot equal the empty sentinel."""
|
||||
|
||||
result = cute.make_rmem_tensor(cute.make_layout((4,)), Uint32)
|
||||
for i in cutlass.range_constexpr(4):
|
||||
word = packed[i]
|
||||
low = Uint16(word & Uint32(0xFFFF))
|
||||
high = Uint16(word >> Uint32(16))
|
||||
if low == Uint16(NEG_ZERO_BF16_BITS):
|
||||
word = word & Uint32(0xFFFF0000)
|
||||
if high == Uint16(NEG_ZERO_BF16_BITS):
|
||||
word = word & Uint32(0x0000FFFF)
|
||||
result[i] = word
|
||||
return result.load()
|
||||
|
||||
|
||||
@cute.jit
|
||||
def fragment_is_dirty(packed):
|
||||
"""Bit-exact upstream sentinel check: one comparison per 32-bit word."""
|
||||
|
||||
dirty = packed[0] == Uint32(NEG_ZERO_F32_BITS)
|
||||
for i in cutlass.range_constexpr(1, 4):
|
||||
dirty = dirty | (packed[i] == Uint32(NEG_ZERO_F32_BITS))
|
||||
return dirty
|
||||
|
||||
|
||||
@cute.jit
|
||||
def warp_sum_specialized(
|
||||
value: Float32,
|
||||
warp_idx: Int32,
|
||||
lane: Int32,
|
||||
warps: cutlass.Constexpr[int],
|
||||
last_warp_lanes: cutlass.Constexpr[int],
|
||||
last_warp_mask: cutlass.Constexpr[int],
|
||||
) -> Float32:
|
||||
"""Warp sum supporting a compile-time partial final warp."""
|
||||
|
||||
if warp_idx == Int32(warps - 1) and cutlass.const_expr(last_warp_lanes < 32):
|
||||
for offset in cutlass.range_constexpr(16, 0, -1):
|
||||
# range_constexpr does not provide powers-of-two stepping.
|
||||
if cutlass.const_expr(offset in (16, 8, 4, 2, 1)):
|
||||
other = cute.arch.shuffle_sync_bfly(
|
||||
value,
|
||||
offset=offset,
|
||||
mask=last_warp_mask,
|
||||
mask_and_clamp=31,
|
||||
)
|
||||
if (lane ^ Int32(offset)) < Int32(last_warp_lanes):
|
||||
value = value + other
|
||||
else:
|
||||
for offset in cutlass.range_constexpr(16, 0, -1):
|
||||
if cutlass.const_expr(offset in (16, 8, 4, 2, 1)):
|
||||
value = value + cute.arch.shuffle_sync_bfly(
|
||||
value,
|
||||
offset=offset,
|
||||
mask=-1,
|
||||
mask_and_clamp=31,
|
||||
)
|
||||
return value
|
||||
|
||||
|
||||
@cute.jit
|
||||
def block_sum_specialized(
|
||||
value: Float32,
|
||||
warp_sums: cute.Tensor,
|
||||
tidx: Int32,
|
||||
warps: cutlass.Constexpr[int],
|
||||
last_warp_lanes: cutlass.Constexpr[int],
|
||||
last_warp_mask: cutlass.Constexpr[int],
|
||||
) -> Float32:
|
||||
"""Upstream-equivalent FP32 block reduction."""
|
||||
|
||||
lane = cute.arch.lane_idx()
|
||||
warp_idx = cute.arch.warp_idx()
|
||||
value = warp_sum_specialized(
|
||||
value, warp_idx, lane, warps, last_warp_lanes, last_warp_mask
|
||||
)
|
||||
if lane == 0:
|
||||
warp_sums[warp_idx] = value
|
||||
cute.arch.barrier()
|
||||
|
||||
block_sum = Float32(0.0)
|
||||
if warp_idx == 0:
|
||||
if lane < Int32(warps):
|
||||
block_sum = warp_sums[lane]
|
||||
block_sum = cute.arch.warp_reduction_sum(block_sum)
|
||||
if lane == 0:
|
||||
warp_sums[0] = block_sum
|
||||
cute.arch.barrier()
|
||||
return warp_sums[0]
|
||||
@@ -0,0 +1,244 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Fused MLA prefill and decode epilogues for Kimi-K3.
|
||||
|
||||
Thin wrappers over the CUDA ops in
|
||||
``csrc/libtorch_stable/fused_kimi_k3_mla_key_concat_kv_cache_kernel.cu``, which
|
||||
mirror ``fused_deepseek_v4_qnorm_rope_kv_rope_full_cache_{bf16,fp8}_insert``.
|
||||
|
||||
- ``fused_mla_key_concat_kv_cache_insert`` (bf16): optionally apply RoPE,
|
||||
concat the full per-head key ``[k_nope | k_pe]`` into ``k_out``, and insert
|
||||
the latent ``[kv_c_normed | k_pe]`` into the paged cache.
|
||||
- ``fused_mla_qkv_quant_kv_cache_fp8_insert`` (fp8): additionally quantize
|
||||
``q``/``k``/``v`` to E4M3 with ``q_scale`` / ``k_scale`` / ``v_scale`` (the
|
||||
cache shares ``k_scale``, as in ``concat_and_cache_mla``).
|
||||
|
||||
The optional ``positions`` / ``cos_sin_cache`` pair enables GPT-J-style RoPE
|
||||
inside the epilogue. Omitting both keeps the K3 NoPE fast path. The kernels use
|
||||
Programmatic Dependent Launch to overlap the tail of the producing GEMMs on
|
||||
sm_90+.
|
||||
"""
|
||||
|
||||
import torch
|
||||
|
||||
|
||||
def fused_mla_key_concat_kv_cache_insert(
|
||||
q: torch.Tensor, # [Tp, H, qk_head_dim], RoPE is applied in place
|
||||
k_nope: torch.Tensor, # [Tp, H, qk_nope_head_dim]
|
||||
k_pe: torch.Tensor, # [Tp, qk_rope_head_dim] or [Tp, 1, qk_rope_head_dim]
|
||||
kv_c_normed: torch.Tensor, # [Tp, kv_lora_rank]
|
||||
kv_cache: torch.Tensor, # [num_blocks, block_size, kv_lora_rank + rope]
|
||||
slot_mapping: torch.Tensor, # [Tp] int64
|
||||
positions: torch.Tensor | None = None, # [Tp] int64
|
||||
cos_sin_cache: torch.Tensor | None = None, # [max_position, rope]
|
||||
) -> torch.Tensor:
|
||||
"""Apply optional RoPE, concat K, and insert the paged latent (bf16).
|
||||
|
||||
Returns the full key ``[Tp, H, qk_nope_head_dim + qk_rope_head_dim]``;
|
||||
optionally rotates ``q`` and writes ``kv_cache`` in place.
|
||||
"""
|
||||
k_pe = k_pe.reshape(k_pe.shape[0], -1)
|
||||
tp, num_heads, qk_nope_head_dim = k_nope.shape
|
||||
qk_head_dim = qk_nope_head_dim + k_pe.shape[1]
|
||||
k_out = torch.empty(
|
||||
(tp, num_heads, qk_head_dim), dtype=k_nope.dtype, device=k_nope.device
|
||||
)
|
||||
if tp == 0:
|
||||
return k_out
|
||||
torch.ops._C.fused_kimi_k3_mla_key_concat_kv_cache_insert(
|
||||
q,
|
||||
k_nope,
|
||||
k_pe,
|
||||
kv_c_normed,
|
||||
k_out,
|
||||
kv_cache,
|
||||
slot_mapping,
|
||||
kv_cache.shape[1],
|
||||
positions,
|
||||
cos_sin_cache,
|
||||
)
|
||||
return k_out
|
||||
|
||||
|
||||
def fused_mla_key_concat_ds_mla_insert(
|
||||
q: torch.Tensor, # [Tp, H, qk_head_dim], RoPE is applied in place
|
||||
k_nope: torch.Tensor, # [Tp, H, qk_nope_head_dim]
|
||||
k_pe: torch.Tensor, # [Tp, qk_rope_head_dim] or [Tp, 1, qk_rope_head_dim]
|
||||
kv_c_normed: torch.Tensor, # [Tp, kv_lora_rank]
|
||||
kv_cache: torch.Tensor, # [num_blocks, block_size, 656] uint8 (fp8_ds_mla)
|
||||
slot_mapping: torch.Tensor, # [Tp] int64
|
||||
positions: torch.Tensor | None = None, # [Tp] int64
|
||||
cos_sin_cache: torch.Tensor | None = None, # [max_position, rope]
|
||||
) -> torch.Tensor:
|
||||
"""Concat full K (bf16) and insert the latent in the fp8_ds_mla layout.
|
||||
|
||||
The cache uses DeepSeek's 656-byte block-scaled layout (NoPE in 4 tiles of
|
||||
128 with per-tile dynamic fp8 scales, RoPE as bf16) -- self-scaling, so no
|
||||
scale argument. Returns the bf16 full key; optionally rotates ``q`` and
|
||||
writes ``kv_cache`` in place.
|
||||
"""
|
||||
k_pe = k_pe.reshape(k_pe.shape[0], -1)
|
||||
tp, num_heads, qk_nope_head_dim = k_nope.shape
|
||||
qk_head_dim = qk_nope_head_dim + k_pe.shape[1]
|
||||
k_out = torch.empty(
|
||||
(tp, num_heads, qk_head_dim), dtype=k_nope.dtype, device=k_nope.device
|
||||
)
|
||||
if tp == 0:
|
||||
return k_out
|
||||
torch.ops._C.fused_kimi_k3_mla_key_concat_ds_mla_insert(
|
||||
q,
|
||||
k_nope,
|
||||
k_pe,
|
||||
kv_c_normed,
|
||||
k_out,
|
||||
kv_cache,
|
||||
slot_mapping,
|
||||
kv_cache.shape[1],
|
||||
positions,
|
||||
cos_sin_cache,
|
||||
)
|
||||
return k_out
|
||||
|
||||
|
||||
def fused_mla_qkv_quant_kv_cache_fp8_insert(
|
||||
q: torch.Tensor, # [Tp, H, qk_head_dim]
|
||||
k_nope: torch.Tensor, # [Tp, H, qk_nope_head_dim]
|
||||
k_pe: torch.Tensor, # [Tp, qk_rope_head_dim] or [Tp, 1, qk_rope_head_dim]
|
||||
kv_c_normed: torch.Tensor, # [Tp, kv_lora_rank]
|
||||
v: torch.Tensor, # [Tp, H, v_head_dim]
|
||||
kv_cache: torch.Tensor, # [num_blocks, block_size, kv_lora_rank + rope] fp8
|
||||
slot_mapping: torch.Tensor, # [Tp] int64
|
||||
q_scale_inv: torch.Tensor, # scalar fp32, 1 / q scale (attention query)
|
||||
k_scale_inv: torch.Tensor, # scalar fp32, 1 / k scale (attention key)
|
||||
v_scale_inv: torch.Tensor, # scalar fp32, 1 / v scale (attention value)
|
||||
cache_scale_inv: torch.Tensor, # scalar fp32, 1 / kv scale (cache latent)
|
||||
positions: torch.Tensor | None = None, # [Tp] int64
|
||||
cos_sin_cache: torch.Tensor | None = None, # [max_position, rope]
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
"""Quantize q/k/v to fp8 and insert the fp8 latent into the paged cache.
|
||||
|
||||
The attention key ``k_fp8`` and the cache latent use *separate* scales
|
||||
(``k_scale_inv`` vs ``cache_scale_inv``): the cache must be quantized with
|
||||
``_k_scale`` (read back by decode / context), while the prefill attention
|
||||
q/k/v currently stay unscaled (the prefill flash path does not dequantize).
|
||||
|
||||
Returns ``(q_fp8, k_fp8, v_fp8)``; writes the fp8 ``kv_cache`` in place.
|
||||
"""
|
||||
k_pe = k_pe.reshape(k_pe.shape[0], -1)
|
||||
tp, num_heads, _ = q.shape
|
||||
qk_head_dim = q.shape[2]
|
||||
v_head_dim = v.shape[2]
|
||||
fp8 = torch.float8_e4m3fn
|
||||
q_fp8 = torch.empty((tp, num_heads, qk_head_dim), dtype=fp8, device=q.device)
|
||||
k_fp8 = torch.empty((tp, num_heads, qk_head_dim), dtype=fp8, device=q.device)
|
||||
v_fp8 = torch.empty((tp, num_heads, v_head_dim), dtype=fp8, device=q.device)
|
||||
if tp == 0:
|
||||
return q_fp8, k_fp8, v_fp8
|
||||
torch.ops._C.fused_kimi_k3_mla_qkv_quant_kv_cache_fp8_insert(
|
||||
q,
|
||||
k_nope,
|
||||
k_pe,
|
||||
kv_c_normed,
|
||||
v,
|
||||
q_fp8,
|
||||
k_fp8,
|
||||
v_fp8,
|
||||
kv_cache,
|
||||
slot_mapping,
|
||||
q_scale_inv,
|
||||
k_scale_inv,
|
||||
v_scale_inv,
|
||||
cache_scale_inv,
|
||||
kv_cache.shape[1],
|
||||
positions,
|
||||
cos_sin_cache,
|
||||
)
|
||||
return q_fp8, k_fp8, v_fp8
|
||||
|
||||
|
||||
def fused_mla_decode_q_concat_kv_cache_insert(
|
||||
ql_nope: torch.Tensor, # [B, H, kv_lora_rank] (BMM1 output, absorbed q)
|
||||
q_pe: torch.Tensor, # [B, H, qk_rope_head_dim]
|
||||
kv_c_normed: torch.Tensor, # [B, kv_lora_rank]
|
||||
k_pe: torch.Tensor, # [B, qk_rope_head_dim] or [B, 1, qk_rope_head_dim]
|
||||
kv_cache: torch.Tensor, # [num_blocks, block_size, entry]
|
||||
slot_mapping: torch.Tensor, # [B] int64
|
||||
*,
|
||||
ds_mla: bool = False,
|
||||
q_scale_inv: torch.Tensor | None = None, # scalar fp32, 1 / q scale
|
||||
cache_scale_inv: torch.Tensor | None = None, # scalar fp32, 1 / kv scale
|
||||
positions: torch.Tensor | None = None, # [B] int64
|
||||
cos_sin_cache: torch.Tensor | None = None, # [max_position, rope]
|
||||
) -> torch.Tensor:
|
||||
"""Concat the latent decode query ``mqa_q = [ql_nope | q_pe]`` and insert the
|
||||
latent ``[kv_c_normed | k_pe]`` into the paged cache, in one launch (runs
|
||||
right before ``forward_mqa``).
|
||||
|
||||
Dispatched by cache format:
|
||||
- bf16 -> bf16 mqa_q, bf16 cache
|
||||
- plain fp8 -> fp8 mqa_q (q_scale_inv), fp8 cache (cache_scale_inv)
|
||||
- fp8_ds_mla -> bf16 mqa_q, 656B block-scaled cache
|
||||
|
||||
Returns ``mqa_q`` of shape ``[B, H, kv_lora_rank + qk_rope_head_dim]``;
|
||||
writes ``kv_cache`` in place.
|
||||
"""
|
||||
k_pe = k_pe.reshape(k_pe.shape[0], -1)
|
||||
b, num_heads, kv_lora_rank = ql_nope.shape
|
||||
entry = kv_lora_rank + q_pe.shape[-1]
|
||||
fp8_q = q_scale_inv is not None
|
||||
out_dtype = torch.float8_e4m3fn if fp8_q else ql_nope.dtype
|
||||
mqa_q = torch.empty((b, num_heads, entry), dtype=out_dtype, device=ql_nope.device)
|
||||
if b == 0:
|
||||
return mqa_q
|
||||
|
||||
if ds_mla:
|
||||
cache = (
|
||||
kv_cache if kv_cache.dtype == torch.uint8 else kv_cache.view(torch.uint8)
|
||||
)
|
||||
torch.ops._C.fused_kimi_k3_mla_decode_q_concat_ds_mla_insert(
|
||||
ql_nope,
|
||||
q_pe,
|
||||
kv_c_normed,
|
||||
k_pe,
|
||||
mqa_q,
|
||||
cache,
|
||||
slot_mapping,
|
||||
cache.shape[1],
|
||||
positions,
|
||||
cos_sin_cache,
|
||||
)
|
||||
elif fp8_q:
|
||||
assert cache_scale_inv is not None, "fp8 decode requires cache_scale_inv"
|
||||
cache = (
|
||||
kv_cache
|
||||
if kv_cache.dtype == torch.float8_e4m3fn
|
||||
else kv_cache.view(torch.float8_e4m3fn)
|
||||
)
|
||||
torch.ops._C.fused_kimi_k3_mla_decode_q_concat_kv_cache_fp8_insert(
|
||||
ql_nope,
|
||||
q_pe,
|
||||
kv_c_normed,
|
||||
k_pe,
|
||||
mqa_q,
|
||||
cache,
|
||||
slot_mapping,
|
||||
q_scale_inv,
|
||||
cache_scale_inv,
|
||||
cache.shape[1],
|
||||
positions,
|
||||
cos_sin_cache,
|
||||
)
|
||||
else:
|
||||
torch.ops._C.fused_kimi_k3_mla_decode_q_concat_kv_cache_insert(
|
||||
ql_nope,
|
||||
q_pe,
|
||||
kv_c_normed,
|
||||
k_pe,
|
||||
mqa_q,
|
||||
kv_cache,
|
||||
slot_mapping,
|
||||
kv_cache.shape[1],
|
||||
positions,
|
||||
cos_sin_cache,
|
||||
)
|
||||
return mqa_q
|
||||
@@ -0,0 +1,265 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from dataclasses import dataclass
|
||||
from functools import partial
|
||||
from typing import ClassVar
|
||||
|
||||
import torch
|
||||
import torch.distributed as dist
|
||||
|
||||
from vllm.distributed import get_tp_group
|
||||
from vllm.model_executor.warmup.cutedsl_warmup import (
|
||||
CuTeDSLCompileUnit,
|
||||
register_cutedsl_warmup_provider,
|
||||
)
|
||||
|
||||
_MAX_NUM_TOKENS = 16
|
||||
_SKINNY_MAX_NUM_TOKENS = 5
|
||||
_MMA_TILER_MN = (64, 32)
|
||||
_GEMM_CLUSTER_MN = (1, 8)
|
||||
_B_PRIME_STAGES = 2
|
||||
_COLLECTIVE_TOKEN_CTAS = 8
|
||||
_LAMPORT_COPY_CTAS = 32
|
||||
_LAMPORT_COPY_THREADS = 224
|
||||
_SUPPORTED_TP_SIZES = (8, 16)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class KimiK3LatentMoETailContract:
|
||||
tp_group_id: int
|
||||
tp_size: int
|
||||
device: torch.device
|
||||
dtype: torch.dtype
|
||||
hidden_size: int
|
||||
latent_size: int
|
||||
max_num_tokens: int
|
||||
rms_eps: float
|
||||
|
||||
|
||||
class KimiK3LatentMoETailOp:
|
||||
"""Process-wide cached K3 latent-MoE tail implementation."""
|
||||
|
||||
_instances: ClassVar[
|
||||
dict[KimiK3LatentMoETailContract, "KimiK3LatentMoETailOp"]
|
||||
] = {}
|
||||
|
||||
@classmethod
|
||||
def _contract_and_group(
|
||||
cls,
|
||||
*,
|
||||
hidden_size: int,
|
||||
latent_size: int,
|
||||
dtype: torch.dtype,
|
||||
device: torch.device,
|
||||
rms_eps: float,
|
||||
) -> tuple[KimiK3LatentMoETailContract, dist.ProcessGroup]:
|
||||
tp = get_tp_group()
|
||||
group = tp.device_group
|
||||
device = torch.device(device)
|
||||
tp_device = torch.device(tp.device)
|
||||
if device != tp_device:
|
||||
raise ValueError(
|
||||
f"Input device {device} does not match TP device {tp_device}."
|
||||
)
|
||||
return (
|
||||
KimiK3LatentMoETailContract(
|
||||
tp_group_id=id(group),
|
||||
tp_size=dist.get_world_size(group),
|
||||
device=device,
|
||||
dtype=dtype,
|
||||
hidden_size=hidden_size,
|
||||
latent_size=latent_size,
|
||||
max_num_tokens=_MAX_NUM_TOKENS,
|
||||
rms_eps=float(rms_eps),
|
||||
),
|
||||
group,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def initialize(
|
||||
cls,
|
||||
*,
|
||||
hidden_size: int,
|
||||
latent_size: int,
|
||||
dtype: torch.dtype,
|
||||
device: torch.device,
|
||||
rms_eps: float,
|
||||
) -> "KimiK3LatentMoETailOp":
|
||||
contract, group = cls._contract_and_group(
|
||||
hidden_size=hidden_size,
|
||||
latent_size=latent_size,
|
||||
dtype=dtype,
|
||||
device=device,
|
||||
rms_eps=rms_eps,
|
||||
)
|
||||
op = cls._instances.get(contract)
|
||||
if op is None:
|
||||
op = cls(contract, group)
|
||||
cls._instances[contract] = op
|
||||
return op
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
contract: KimiK3LatentMoETailContract,
|
||||
group: dist.ProcessGroup,
|
||||
) -> None:
|
||||
if contract.tp_size not in _SUPPORTED_TP_SIZES:
|
||||
raise ValueError(
|
||||
"K3 latent-MoE tail fusion requires TP in "
|
||||
f"{_SUPPORTED_TP_SIZES}, got {contract.tp_size}."
|
||||
)
|
||||
if contract.device.type != "cuda":
|
||||
raise ValueError("K3 latent-MoE tail fusion requires a CUDA device.")
|
||||
if torch.cuda.get_device_capability(contract.device)[0] != 10:
|
||||
raise ValueError("K3 latent-MoE tail fusion requires SM100.")
|
||||
if contract.dtype != torch.bfloat16:
|
||||
raise ValueError("K3 latent-MoE tail fusion requires bfloat16.")
|
||||
if (contract.hidden_size, contract.latent_size) != (7168, 3584):
|
||||
raise ValueError(
|
||||
"K3 latent-MoE tail fusion requires hidden_size=7168 and "
|
||||
"latent_size=3584."
|
||||
)
|
||||
|
||||
self.contract = contract
|
||||
self.rank = dist.get_rank(group)
|
||||
from .cute_dsl.latent_moe_tail import (
|
||||
AdaptiveUpProjectionKernel,
|
||||
CollectiveKernel,
|
||||
LamportCopyKernel,
|
||||
)
|
||||
|
||||
with torch.accelerator.device_index(contract.device.index):
|
||||
self._collective = CollectiveKernel(
|
||||
group=group,
|
||||
rank=self.rank,
|
||||
tp_size=contract.tp_size,
|
||||
latent_dim=contract.latent_size,
|
||||
hidden_dim=contract.hidden_size,
|
||||
max_m=contract.max_num_tokens,
|
||||
max_token_ctas=_COLLECTIVE_TOKEN_CTAS,
|
||||
rms_eps=contract.rms_eps,
|
||||
fp32_internal=False,
|
||||
)
|
||||
self._up_projection = AdaptiveUpProjectionKernel(
|
||||
group=group,
|
||||
rank=self.rank,
|
||||
tp_size=contract.tp_size,
|
||||
latent_dim=contract.latent_size,
|
||||
hidden_dim=contract.hidden_size,
|
||||
max_m=contract.max_num_tokens,
|
||||
skinny_max_m=_SKINNY_MAX_NUM_TOKENS,
|
||||
mma_tiler_mn=_MMA_TILER_MN,
|
||||
cluster_shape_mn=_GEMM_CLUSTER_MN,
|
||||
b_prime_stages=_B_PRIME_STAGES,
|
||||
)
|
||||
self._lamport_copy = LamportCopyKernel(
|
||||
hidden_dim=contract.hidden_size,
|
||||
max_m=contract.max_num_tokens,
|
||||
ctas=_LAMPORT_COPY_CTAS,
|
||||
threads=_LAMPORT_COPY_THREADS,
|
||||
)
|
||||
register_cutedsl_warmup_provider(self)
|
||||
|
||||
def get_cutedsl_warmup_compile_units(self) -> tuple[CuTeDSLCompileUnit, ...]:
|
||||
contract = self.contract
|
||||
skinny_units = tuple(
|
||||
CuTeDSLCompileUnit(
|
||||
name="K3 latent MoE tail Skinny up-projection",
|
||||
key=(
|
||||
"k3-latent-moe-tail-skinny-up-projection",
|
||||
contract,
|
||||
m,
|
||||
),
|
||||
compile=partial(self._up_projection.compile_skinny, m),
|
||||
)
|
||||
for m in range(1, self._up_projection.skinny_max_m + 1)
|
||||
)
|
||||
return skinny_units + (
|
||||
CuTeDSLCompileUnit(
|
||||
name="K3 latent MoE tail dynamic up-projection",
|
||||
key=(
|
||||
"k3-latent-moe-tail-dynamic-up-projection",
|
||||
contract,
|
||||
_MMA_TILER_MN,
|
||||
_GEMM_CLUSTER_MN,
|
||||
_B_PRIME_STAGES,
|
||||
),
|
||||
compile=self._up_projection.compile_dynamic,
|
||||
),
|
||||
)
|
||||
|
||||
def __call__(
|
||||
self,
|
||||
routed_output: torch.Tensor,
|
||||
shared_output: torch.Tensor,
|
||||
rms_weight: torch.Tensor,
|
||||
up_weight: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
self._validate_inputs(
|
||||
routed_output,
|
||||
shared_output,
|
||||
rms_weight,
|
||||
up_weight,
|
||||
)
|
||||
self._up_projection.ensure_compiled(routed_output.shape[0])
|
||||
latent, shared_shard = self._collective(
|
||||
routed_output,
|
||||
shared_output,
|
||||
rms_weight,
|
||||
)
|
||||
local_hidden_size = self.contract.hidden_size // self.contract.tp_size
|
||||
local_up_weight = up_weight.narrow(
|
||||
0,
|
||||
self.rank * local_hidden_size,
|
||||
local_hidden_size,
|
||||
)
|
||||
mailbox = self._up_projection(
|
||||
latent,
|
||||
local_up_weight,
|
||||
shared_shard,
|
||||
)
|
||||
return self._lamport_copy(
|
||||
mailbox,
|
||||
m=routed_output.shape[0],
|
||||
).squeeze(0)
|
||||
|
||||
def _validate_inputs(
|
||||
self,
|
||||
routed_output: torch.Tensor,
|
||||
shared_output: torch.Tensor,
|
||||
rms_weight: torch.Tensor,
|
||||
up_weight: torch.Tensor,
|
||||
) -> None:
|
||||
contract = self.contract
|
||||
if routed_output.ndim != 2:
|
||||
raise ValueError("routed_output must be a 2D tensor.")
|
||||
num_tokens = routed_output.shape[0]
|
||||
if routed_output.shape != (num_tokens, contract.latent_size):
|
||||
raise ValueError(
|
||||
f"routed_output must have shape [M, {contract.latent_size}]."
|
||||
)
|
||||
if shared_output.shape != (num_tokens, contract.hidden_size):
|
||||
raise ValueError(
|
||||
f"shared_output must have shape [M, {contract.hidden_size}]."
|
||||
)
|
||||
if rms_weight.shape != (contract.latent_size,):
|
||||
raise ValueError(f"rms_weight must have shape [{contract.latent_size}].")
|
||||
if up_weight.shape != (contract.hidden_size, contract.latent_size):
|
||||
raise ValueError(
|
||||
"up_weight must have shape "
|
||||
f"[{contract.hidden_size}, {contract.latent_size}]."
|
||||
)
|
||||
if not 1 <= num_tokens <= contract.max_num_tokens:
|
||||
raise ValueError(
|
||||
"K3 latent-MoE tail fusion requires between 1 and "
|
||||
f"{contract.max_num_tokens} tokens."
|
||||
)
|
||||
|
||||
tensors = (routed_output, shared_output, rms_weight, up_weight)
|
||||
if any(tensor.device != contract.device for tensor in tensors):
|
||||
raise ValueError("All inputs must be on the contract device.")
|
||||
if any(tensor.dtype != contract.dtype for tensor in tensors):
|
||||
raise ValueError("All inputs must use the contract dtype.")
|
||||
if any(not tensor.is_contiguous() for tensor in tensors):
|
||||
raise ValueError("All inputs must be contiguous.")
|
||||
@@ -0,0 +1,73 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.distributed import (
|
||||
get_tensor_model_parallel_rank,
|
||||
get_tensor_model_parallel_world_size,
|
||||
get_tp_group,
|
||||
tensor_model_parallel_all_gather,
|
||||
tensor_model_parallel_reduce_scatter,
|
||||
)
|
||||
|
||||
|
||||
def _custom_collective(
|
||||
name: str,
|
||||
x: torch.Tensor,
|
||||
) -> torch.Tensor | None:
|
||||
device_communicator = get_tp_group().device_communicator
|
||||
if device_communicator is None:
|
||||
return None
|
||||
collective = getattr(device_communicator, name, None)
|
||||
if collective is None:
|
||||
return None
|
||||
return collective(x)
|
||||
|
||||
|
||||
def sp_all_gather(x: torch.Tensor) -> torch.Tensor:
|
||||
output = _custom_collective("custom_all_gather", x)
|
||||
if output is not None:
|
||||
return output
|
||||
return tensor_model_parallel_all_gather(x, 0)
|
||||
|
||||
|
||||
def sp_reduce_scatter(x: torch.Tensor) -> torch.Tensor:
|
||||
assert x.ndim == 2
|
||||
tp_size = get_tensor_model_parallel_world_size()
|
||||
sp_pad = (-x.shape[0]) % tp_size
|
||||
if sp_pad > 0:
|
||||
x = torch.nn.functional.pad(x, (0, 0, 0, sp_pad))
|
||||
output = _custom_collective("custom_reduce_scatter", x)
|
||||
if output is not None:
|
||||
return output
|
||||
return tensor_model_parallel_reduce_scatter(x, 0)
|
||||
|
||||
|
||||
def sp_shard(x: torch.Tensor) -> torch.Tensor:
|
||||
assert x.ndim == 2
|
||||
tp_size = get_tensor_model_parallel_world_size()
|
||||
tp_rank = get_tensor_model_parallel_rank()
|
||||
sp_pad = (-x.shape[0]) % tp_size
|
||||
if sp_pad > 0:
|
||||
x = torch.nn.functional.pad(x, (0, 0, 0, sp_pad))
|
||||
chunk = x.shape[0] // tp_size
|
||||
return x[tp_rank * chunk : (tp_rank + 1) * chunk]
|
||||
|
||||
|
||||
def sp_padding_mask(
|
||||
is_padding: torch.Tensor | None,
|
||||
hidden_states: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
num_tokens = hidden_states.shape[0]
|
||||
if is_padding is None:
|
||||
is_padding = hidden_states.new_zeros(num_tokens, dtype=torch.bool)
|
||||
assert is_padding.shape[0] == num_tokens
|
||||
|
||||
tp_size = get_tensor_model_parallel_world_size()
|
||||
sp_pad = (-num_tokens) % tp_size
|
||||
if sp_pad > 0:
|
||||
is_padding = torch.nn.functional.pad(is_padding, (0, sp_pad), value=True)
|
||||
chunk = is_padding.shape[0] // tp_size
|
||||
tp_rank = get_tensor_model_parallel_rank()
|
||||
return is_padding[tp_rank * chunk : (tp_rank + 1) * chunk]
|
||||
@@ -0,0 +1,2 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
@@ -0,0 +1,28 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from .chunk import (
|
||||
chunk_kda,
|
||||
chunk_kda_fwd,
|
||||
chunk_kda_with_fused_gate,
|
||||
chunk_kda_with_fused_gate_fwd,
|
||||
fused_kda_gate,
|
||||
fused_kda_gate_chunk_cumsum,
|
||||
)
|
||||
from .fused_recurrent import (
|
||||
fused_recurrent_kda,
|
||||
fused_recurrent_kda_fwd,
|
||||
fused_recurrent_kda_packed_decode,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"chunk_kda",
|
||||
"chunk_kda_fwd",
|
||||
"chunk_kda_with_fused_gate",
|
||||
"chunk_kda_with_fused_gate_fwd",
|
||||
"fused_kda_gate",
|
||||
"fused_kda_gate_chunk_cumsum",
|
||||
"fused_recurrent_kda",
|
||||
"fused_recurrent_kda_fwd",
|
||||
"fused_recurrent_kda_packed_decode",
|
||||
]
|
||||
@@ -0,0 +1,938 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# SPDX-FileCopyrightText: Songlin Yang, Yu Zhang, Zhiyuan Li
|
||||
#
|
||||
# This file contains code copied from the flash-linear-attention project.
|
||||
# The original source code was licensed under the MIT license and included
|
||||
# the following copyright notice:
|
||||
# Copyright (c) 2023-2026, Songlin Yang, Yu Zhang, Zhiyuan Li
|
||||
# ruff: noqa: E501
|
||||
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.third_party.flash_linear_attention.ops.chunk_delta_h import (
|
||||
chunk_gated_delta_rule_fwd_h,
|
||||
)
|
||||
from vllm.third_party.flash_linear_attention.ops.cumsum import chunk_local_cumsum
|
||||
from vllm.third_party.flash_linear_attention.ops.index import prepare_chunk_indices
|
||||
from vllm.third_party.flash_linear_attention.ops.l2norm import l2norm_fwd
|
||||
from vllm.third_party.flash_linear_attention.ops.op import exp2, log
|
||||
from vllm.third_party.flash_linear_attention.ops.utils import FLA_CHUNK_SIZE, is_amd
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.utils.math_utils import RCP_LN2, cdiv, next_power_of_2
|
||||
|
||||
from .chunk_intra import chunk_kda_fwd_intra
|
||||
|
||||
BT_LIST_AUTOTUNE = [32, 64, 128]
|
||||
NUM_WARPS_AUTOTUNE = [2, 4, 8, 16] if is_amd else [4, 8, 16, 32]
|
||||
|
||||
|
||||
@triton.heuristics(
|
||||
{
|
||||
"STORE_QG": lambda args: args["qg"] is not None,
|
||||
"STORE_KG": lambda args: args["kg"] is not None,
|
||||
"IS_VARLEN": lambda args: args["cu_seqlens"] is not None,
|
||||
}
|
||||
)
|
||||
@triton.autotune(
|
||||
configs=[
|
||||
triton.Config({}, num_warps=num_warps, num_stages=num_stages)
|
||||
for num_warps in [2, 4, 8]
|
||||
for num_stages in [2, 3, 4]
|
||||
],
|
||||
key=["H", "K", "V", "BT", "BK", "BV", "IS_VARLEN"],
|
||||
)
|
||||
@triton.jit(do_not_specialize=["T"])
|
||||
def recompute_w_u_fwd_kernel(
|
||||
q,
|
||||
k,
|
||||
qg,
|
||||
kg,
|
||||
v,
|
||||
beta,
|
||||
w,
|
||||
u,
|
||||
A,
|
||||
gk,
|
||||
cu_seqlens,
|
||||
chunk_indices,
|
||||
T,
|
||||
H: tl.constexpr,
|
||||
K: tl.constexpr,
|
||||
V: tl.constexpr,
|
||||
BT: tl.constexpr,
|
||||
BK: tl.constexpr,
|
||||
BV: tl.constexpr,
|
||||
STORE_QG: tl.constexpr,
|
||||
STORE_KG: tl.constexpr,
|
||||
IS_VARLEN: tl.constexpr,
|
||||
DOT_PRECISION: tl.constexpr,
|
||||
):
|
||||
i_t, i_bh = tl.program_id(0), tl.program_id(1)
|
||||
i_b, i_h = i_bh // H, i_bh % H
|
||||
if IS_VARLEN:
|
||||
i_n, i_t = (
|
||||
tl.load(chunk_indices + i_t * 2).to(tl.int32),
|
||||
tl.load(chunk_indices + i_t * 2 + 1).to(tl.int32),
|
||||
)
|
||||
bos, eos = (
|
||||
tl.load(cu_seqlens + i_n).to(tl.int32),
|
||||
tl.load(cu_seqlens + i_n + 1).to(tl.int32),
|
||||
)
|
||||
T = eos - bos
|
||||
else:
|
||||
bos, eos = i_b * T, i_b * T + T
|
||||
p_b = tl.make_block_ptr(beta + bos * H + i_h, (T,), (H,), (i_t * BT,), (BT,), (0,))
|
||||
b_b = tl.load(p_b, boundary_check=(0,)).to(tl.float32)
|
||||
|
||||
p_A = tl.make_block_ptr(
|
||||
A + (bos * H + i_h) * BT, (T, BT), (H * BT, 1), (i_t * BT, 0), (BT, BT), (1, 0)
|
||||
)
|
||||
b_A = tl.load(p_A, boundary_check=(0, 1))
|
||||
|
||||
for i_v in range(tl.cdiv(V, BV)):
|
||||
p_v = tl.make_block_ptr(
|
||||
v + (bos * H + i_h) * V,
|
||||
(T, V),
|
||||
(H * V, 1),
|
||||
(i_t * BT, i_v * BV),
|
||||
(BT, BV),
|
||||
(1, 0),
|
||||
)
|
||||
p_u = tl.make_block_ptr(
|
||||
u + (bos * H + i_h) * V,
|
||||
(T, V),
|
||||
(H * V, 1),
|
||||
(i_t * BT, i_v * BV),
|
||||
(BT, BV),
|
||||
(1, 0),
|
||||
)
|
||||
b_v = tl.load(p_v, boundary_check=(0, 1))
|
||||
b_vb = (b_v * b_b[:, None]).to(b_v.dtype)
|
||||
b_u = tl.dot(b_A, b_vb, input_precision=DOT_PRECISION)
|
||||
tl.store(p_u, b_u.to(p_u.dtype.element_ty), boundary_check=(0, 1))
|
||||
|
||||
for i_k in range(tl.cdiv(K, BK)):
|
||||
p_w = tl.make_block_ptr(
|
||||
w + (bos * H + i_h) * K,
|
||||
(T, K),
|
||||
(H * K, 1),
|
||||
(i_t * BT, i_k * BK),
|
||||
(BT, BK),
|
||||
(1, 0),
|
||||
)
|
||||
p_k = tl.make_block_ptr(
|
||||
k + (bos * H + i_h) * K,
|
||||
(T, K),
|
||||
(H * K, 1),
|
||||
(i_t * BT, i_k * BK),
|
||||
(BT, BK),
|
||||
(1, 0),
|
||||
)
|
||||
b_k = tl.load(p_k, boundary_check=(0, 1))
|
||||
b_kb = b_k * b_b[:, None]
|
||||
|
||||
p_gk = tl.make_block_ptr(
|
||||
gk + (bos * H + i_h) * K,
|
||||
(T, K),
|
||||
(H * K, 1),
|
||||
(i_t * BT, i_k * BK),
|
||||
(BT, BK),
|
||||
(1, 0),
|
||||
)
|
||||
b_gk = tl.load(p_gk, boundary_check=(0, 1))
|
||||
b_kb *= exp2(b_gk)
|
||||
if STORE_QG:
|
||||
p_q = tl.make_block_ptr(
|
||||
q + (bos * H + i_h) * K,
|
||||
(T, K),
|
||||
(H * K, 1),
|
||||
(i_t * BT, i_k * BK),
|
||||
(BT, BK),
|
||||
(1, 0),
|
||||
)
|
||||
p_qg = tl.make_block_ptr(
|
||||
qg + (bos * H + i_h) * K,
|
||||
(T, K),
|
||||
(H * K, 1),
|
||||
(i_t * BT, i_k * BK),
|
||||
(BT, BK),
|
||||
(1, 0),
|
||||
)
|
||||
b_q = tl.load(p_q, boundary_check=(0, 1))
|
||||
b_qg = b_q * exp2(b_gk)
|
||||
tl.store(p_qg, b_qg.to(p_qg.dtype.element_ty), boundary_check=(0, 1))
|
||||
if STORE_KG:
|
||||
last_idx = min(i_t * BT + BT, T) - 1
|
||||
|
||||
o_k = i_k * BK + tl.arange(0, BK)
|
||||
m_k = o_k < K
|
||||
b_gn = tl.load(
|
||||
gk + ((bos + last_idx) * H + i_h) * K + o_k, mask=m_k, other=0.0
|
||||
)
|
||||
b_kg = b_k * exp2(b_gn - b_gk)
|
||||
|
||||
p_kg = tl.make_block_ptr(
|
||||
kg + (bos * H + i_h) * K,
|
||||
(T, K),
|
||||
(H * K, 1),
|
||||
(i_t * BT, i_k * BK),
|
||||
(BT, BK),
|
||||
(1, 0),
|
||||
)
|
||||
tl.store(p_kg, b_kg.to(p_kg.dtype.element_ty), boundary_check=(0, 1))
|
||||
|
||||
b_w = tl.dot(b_A, b_kb.to(b_k.dtype))
|
||||
tl.store(p_w, b_w.to(p_w.dtype.element_ty), boundary_check=(0, 1))
|
||||
|
||||
|
||||
def recompute_w_u_fwd(
|
||||
k: torch.Tensor,
|
||||
v: torch.Tensor,
|
||||
beta: torch.Tensor,
|
||||
A: torch.Tensor,
|
||||
q: torch.Tensor | None = None,
|
||||
gk: torch.Tensor | None = None,
|
||||
cu_seqlens: torch.Tensor | None = None,
|
||||
chunk_indices: torch.Tensor | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
B, T, H, K, V = *k.shape, v.shape[-1]
|
||||
BT = A.shape[-1]
|
||||
BK = 64
|
||||
BV = 64
|
||||
|
||||
if chunk_indices is None and cu_seqlens is not None:
|
||||
chunk_indices = prepare_chunk_indices(cu_seqlens, BT)
|
||||
NT = cdiv(T, BT) if cu_seqlens is None else len(chunk_indices)
|
||||
|
||||
w = torch.empty_like(k)
|
||||
u = torch.empty_like(v)
|
||||
kg = torch.empty_like(k) if gk is not None else None
|
||||
recompute_w_u_fwd_kernel[(NT, B * H)](
|
||||
q=q,
|
||||
k=k,
|
||||
qg=None,
|
||||
kg=kg,
|
||||
v=v,
|
||||
beta=beta,
|
||||
w=w,
|
||||
u=u,
|
||||
A=A,
|
||||
gk=gk,
|
||||
cu_seqlens=cu_seqlens,
|
||||
chunk_indices=chunk_indices,
|
||||
T=T,
|
||||
H=H,
|
||||
K=K,
|
||||
V=V,
|
||||
BT=BT,
|
||||
BK=BK,
|
||||
BV=BV,
|
||||
DOT_PRECISION="ieee",
|
||||
)
|
||||
return w, u, None, kg
|
||||
|
||||
|
||||
@triton.heuristics({"IS_VARLEN": lambda args: args["cu_seqlens"] is not None})
|
||||
@triton.autotune(
|
||||
configs=[
|
||||
triton.Config({"BK": BK, "BV": BV}, num_warps=num_warps, num_stages=num_stages)
|
||||
for BK in [32, 64]
|
||||
for BV in [64, 128]
|
||||
for num_warps in [2, 4, 8]
|
||||
for num_stages in [2, 3, 4]
|
||||
],
|
||||
key=["BT"],
|
||||
)
|
||||
@triton.jit(do_not_specialize=["T"])
|
||||
def chunk_gla_fwd_kernel_o(
|
||||
q,
|
||||
v,
|
||||
g,
|
||||
h,
|
||||
o,
|
||||
A,
|
||||
cu_seqlens,
|
||||
chunk_indices,
|
||||
scale,
|
||||
T,
|
||||
H: tl.constexpr,
|
||||
K: tl.constexpr,
|
||||
V: tl.constexpr,
|
||||
BT: tl.constexpr,
|
||||
BK: tl.constexpr,
|
||||
BV: tl.constexpr,
|
||||
IS_VARLEN: tl.constexpr,
|
||||
):
|
||||
i_v, i_t, i_bh = tl.program_id(0), tl.program_id(1), tl.program_id(2)
|
||||
i_b, i_h = i_bh // H, i_bh % H
|
||||
if IS_VARLEN:
|
||||
i_tg = i_t
|
||||
i_n, i_t = (
|
||||
tl.load(chunk_indices + i_t * 2).to(tl.int32),
|
||||
tl.load(chunk_indices + i_t * 2 + 1).to(tl.int32),
|
||||
)
|
||||
bos, eos = (
|
||||
tl.load(cu_seqlens + i_n).to(tl.int32),
|
||||
tl.load(cu_seqlens + i_n + 1).to(tl.int32),
|
||||
)
|
||||
T = eos - bos
|
||||
NT = tl.cdiv(T, BT)
|
||||
else:
|
||||
NT = tl.cdiv(T, BT)
|
||||
i_tg = i_b * NT + i_t
|
||||
bos, eos = i_b * T, i_b * T + T
|
||||
|
||||
m_s = tl.arange(0, BT)[:, None] >= tl.arange(0, BT)[None, :]
|
||||
|
||||
b_o = tl.zeros([BT, BV], dtype=tl.float32)
|
||||
for i_k in range(tl.cdiv(K, BK)):
|
||||
p_q = tl.make_block_ptr(
|
||||
q + (bos * H + i_h) * K,
|
||||
(T, K),
|
||||
(H * K, 1),
|
||||
(i_t * BT, i_k * BK),
|
||||
(BT, BK),
|
||||
(1, 0),
|
||||
)
|
||||
p_g = tl.make_block_ptr(
|
||||
g + (bos * H + i_h) * K,
|
||||
(T, K),
|
||||
(H * K, 1),
|
||||
(i_t * BT, i_k * BK),
|
||||
(BT, BK),
|
||||
(1, 0),
|
||||
)
|
||||
p_h = tl.make_block_ptr(
|
||||
h + (i_tg * H + i_h) * K * V,
|
||||
(V, K),
|
||||
(K, 1),
|
||||
(i_v * BV, i_k * BK),
|
||||
(BV, BK),
|
||||
(1, 0),
|
||||
)
|
||||
|
||||
# [BT, BK]
|
||||
b_q = tl.load(p_q, boundary_check=(0, 1))
|
||||
b_q = (b_q * scale).to(b_q.dtype)
|
||||
# [BT, BK]
|
||||
b_g = tl.load(p_g, boundary_check=(0, 1))
|
||||
# [BT, BK]
|
||||
b_qg = (b_q * exp2(b_g)).to(b_q.dtype)
|
||||
# [BV, BK]
|
||||
b_h = tl.load(p_h, boundary_check=(0, 1))
|
||||
# [BT, BV]
|
||||
if i_k >= 0:
|
||||
b_o += tl.dot(b_qg, tl.trans(b_h).to(b_qg.dtype))
|
||||
p_v = tl.make_block_ptr(
|
||||
v + (bos * H + i_h) * V,
|
||||
(T, V),
|
||||
(H * V, 1),
|
||||
(i_t * BT, i_v * BV),
|
||||
(BT, BV),
|
||||
(1, 0),
|
||||
)
|
||||
p_o = tl.make_block_ptr(
|
||||
o + (bos * H + i_h) * V,
|
||||
(T, V),
|
||||
(H * V, 1),
|
||||
(i_t * BT, i_v * BV),
|
||||
(BT, BV),
|
||||
(1, 0),
|
||||
)
|
||||
p_A = tl.make_block_ptr(
|
||||
A + (bos * H + i_h) * BT, (T, BT), (H * BT, 1), (i_t * BT, 0), (BT, BT), (1, 0)
|
||||
)
|
||||
# [BT, BV]
|
||||
b_v = tl.load(p_v, boundary_check=(0, 1))
|
||||
# [BT, BT]
|
||||
b_A = tl.load(p_A, boundary_check=(0, 1))
|
||||
b_A = tl.where(m_s, b_A, 0.0).to(b_v.dtype)
|
||||
b_o += tl.dot(b_A, b_v, allow_tf32=False)
|
||||
tl.store(p_o, b_o.to(p_o.dtype.element_ty), boundary_check=(0, 1))
|
||||
|
||||
|
||||
def chunk_gla_fwd_o_gk(
|
||||
q: torch.Tensor,
|
||||
v: torch.Tensor,
|
||||
g: torch.Tensor,
|
||||
A: torch.Tensor,
|
||||
h: torch.Tensor,
|
||||
o: torch.Tensor,
|
||||
scale: float,
|
||||
cu_seqlens: torch.Tensor | None = None,
|
||||
chunk_indices: torch.Tensor | None = None,
|
||||
chunk_size: int = FLA_CHUNK_SIZE,
|
||||
):
|
||||
B, T, H, K, V = *q.shape, v.shape[-1]
|
||||
BT = chunk_size
|
||||
|
||||
if chunk_indices is None and cu_seqlens is not None:
|
||||
chunk_indices = prepare_chunk_indices(cu_seqlens, chunk_size)
|
||||
NT = cdiv(T, BT) if cu_seqlens is None else len(chunk_indices)
|
||||
|
||||
def grid(meta):
|
||||
return (cdiv(V, meta["BV"]), NT, B * H)
|
||||
|
||||
chunk_gla_fwd_kernel_o[grid](
|
||||
q=q,
|
||||
v=v,
|
||||
g=g,
|
||||
h=h,
|
||||
o=o,
|
||||
A=A,
|
||||
cu_seqlens=cu_seqlens,
|
||||
chunk_indices=chunk_indices,
|
||||
scale=scale,
|
||||
T=T,
|
||||
H=H,
|
||||
K=K,
|
||||
V=V,
|
||||
BT=BT,
|
||||
)
|
||||
return o
|
||||
|
||||
|
||||
@triton.heuristics(
|
||||
{
|
||||
"HAS_BIAS": lambda args: args["g_bias"] is not None,
|
||||
"IS_VARLEN": lambda args: args["cu_seqlens"] is not None,
|
||||
}
|
||||
)
|
||||
@triton.autotune(
|
||||
configs=[
|
||||
triton.Config({"BS": BS}, num_warps=num_warps)
|
||||
for BS in [32, 64]
|
||||
for num_warps in [2, 4, 8]
|
||||
],
|
||||
key=["H", "S", "BT", "IS_VARLEN"],
|
||||
)
|
||||
@triton.jit(do_not_specialize=["T"])
|
||||
def kda_gate_chunk_cumsum_vector_kernel(
|
||||
s,
|
||||
raw_beta,
|
||||
A_log,
|
||||
g_bias,
|
||||
o,
|
||||
beta_out,
|
||||
cu_seqlens,
|
||||
chunk_indices,
|
||||
cumsum_scale,
|
||||
lower_bound,
|
||||
beta,
|
||||
threshold,
|
||||
T,
|
||||
stride_beta_batch,
|
||||
stride_beta_token,
|
||||
stride_beta_head,
|
||||
H: tl.constexpr,
|
||||
S: tl.constexpr,
|
||||
BT: tl.constexpr,
|
||||
BS: tl.constexpr,
|
||||
HAS_BIAS: tl.constexpr,
|
||||
IS_VARLEN: tl.constexpr,
|
||||
USE_LOWER_BOUND: tl.constexpr,
|
||||
):
|
||||
i_s, i_t, i_bh = tl.program_id(0), tl.program_id(1), tl.program_id(2)
|
||||
i_b, i_h = i_bh // H, i_bh % H
|
||||
if IS_VARLEN:
|
||||
i_n, i_t = (
|
||||
tl.load(chunk_indices + i_t * 2).to(tl.int32),
|
||||
tl.load(chunk_indices + i_t * 2 + 1).to(tl.int32),
|
||||
)
|
||||
bos, eos = (
|
||||
tl.load(cu_seqlens + i_n).to(tl.int32),
|
||||
tl.load(cu_seqlens + i_n + 1).to(tl.int32),
|
||||
)
|
||||
T = eos - bos
|
||||
else:
|
||||
bos = i_b * T
|
||||
|
||||
if i_s == 0:
|
||||
o_beta_t = tl.arange(0, BT)
|
||||
m_beta = i_t * BT + o_beta_t < T
|
||||
if IS_VARLEN:
|
||||
p_beta = (
|
||||
raw_beta
|
||||
+ (bos + i_t * BT + o_beta_t) * stride_beta_token
|
||||
+ i_h * stride_beta_head
|
||||
)
|
||||
else:
|
||||
p_beta = (
|
||||
raw_beta
|
||||
+ i_b * stride_beta_batch
|
||||
+ (i_t * BT + o_beta_t) * stride_beta_token
|
||||
+ i_h * stride_beta_head
|
||||
)
|
||||
b_beta = tl.load(p_beta, mask=m_beta, other=0.0).to(tl.float32)
|
||||
p_beta_out = beta_out + (bos + i_t * BT + o_beta_t) * H + i_h
|
||||
tl.store(p_beta_out, tl.sigmoid(b_beta), mask=m_beta)
|
||||
return
|
||||
|
||||
i_s -= 1
|
||||
|
||||
p_s = tl.make_block_ptr(
|
||||
s + (bos * H + i_h) * S,
|
||||
(T, S),
|
||||
(H * S, 1),
|
||||
(i_t * BT, i_s * BS),
|
||||
(BT, BS),
|
||||
(1, 0),
|
||||
)
|
||||
p_o = tl.make_block_ptr(
|
||||
o + (bos * H + i_h) * S,
|
||||
(T, S),
|
||||
(H * S, 1),
|
||||
(i_t * BT, i_s * BS),
|
||||
(BT, BS),
|
||||
(1, 0),
|
||||
)
|
||||
|
||||
b_s = tl.load(p_s, boundary_check=(0, 1)).to(tl.float32)
|
||||
if HAS_BIAS:
|
||||
p_bias = tl.make_block_ptr(
|
||||
g_bias + i_h * S,
|
||||
(S,),
|
||||
(1,),
|
||||
(i_s * BS,),
|
||||
(BS,),
|
||||
(0,),
|
||||
)
|
||||
b_bias = tl.load(p_bias, boundary_check=(0,)).to(tl.float32)
|
||||
b_s += b_bias[None, :]
|
||||
|
||||
b_a = tl.exp(tl.load(A_log + i_h).to(tl.float32))
|
||||
if USE_LOWER_BOUND:
|
||||
b_gate = lower_bound * tl.sigmoid(b_a * b_s)
|
||||
else:
|
||||
b_g_scaled = b_s * beta
|
||||
b_softplus = tl.where(
|
||||
b_g_scaled > threshold,
|
||||
b_s,
|
||||
(1.0 / beta) * log(1.0 + tl.exp(b_g_scaled)),
|
||||
)
|
||||
b_gate = -b_a * b_softplus
|
||||
|
||||
# Boundary loads return zero, but bias and gate activation can make padded
|
||||
# rows nonzero. Padding trails valid rows, so it only affects masked stores.
|
||||
b_o = tl.cumsum(b_gate, axis=0) * cumsum_scale
|
||||
tl.store(p_o, b_o.to(p_o.dtype.element_ty), boundary_check=(0, 1))
|
||||
|
||||
|
||||
def fused_kda_gate_chunk_cumsum(
|
||||
raw_g: torch.Tensor,
|
||||
raw_beta: torch.Tensor,
|
||||
A_log: torch.Tensor,
|
||||
g_bias: torch.Tensor | None = None,
|
||||
beta: float = 1.0,
|
||||
threshold: float = 20.0,
|
||||
lower_bound: float | None = None,
|
||||
cu_seqlens: torch.Tensor | None = None,
|
||||
chunk_indices: torch.Tensor | None = None,
|
||||
chunk_size: int = FLA_CHUNK_SIZE,
|
||||
output_dtype: torch.dtype | None = torch.float,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
if cu_seqlens is not None:
|
||||
assert raw_g.shape[0] == 1, (
|
||||
"Only batch size 1 is supported when cu_seqlens are provided"
|
||||
)
|
||||
B, T, H, D = raw_g.shape
|
||||
if raw_beta.shape != (B, T, H):
|
||||
raise ValueError(
|
||||
f"Expected raw_beta shape {(B, T, H)}, got {raw_beta.shape}"
|
||||
)
|
||||
if chunk_indices is None and cu_seqlens is not None:
|
||||
chunk_indices = prepare_chunk_indices(cu_seqlens, chunk_size)
|
||||
NT = cdiv(T, chunk_size) if cu_seqlens is None else len(chunk_indices)
|
||||
|
||||
A_log = A_log.reshape(-1)
|
||||
if g_bias is not None:
|
||||
g_bias = g_bias.reshape(-1)
|
||||
y = torch.empty_like(raw_g, dtype=output_dtype or raw_g.dtype)
|
||||
beta_out = torch.empty(raw_beta.shape, device=raw_beta.device, dtype=torch.float32)
|
||||
|
||||
def grid(meta):
|
||||
# For each (chunk, head), program 0 computes beta without extending a
|
||||
# gate tile's critical path. The remaining programs cover the gate dim.
|
||||
return (cdiv(meta["S"], meta["BS"]) + 1, NT, B * H)
|
||||
|
||||
kda_gate_chunk_cumsum_vector_kernel[grid](
|
||||
s=raw_g,
|
||||
raw_beta=raw_beta,
|
||||
A_log=A_log,
|
||||
g_bias=g_bias,
|
||||
o=y,
|
||||
beta_out=beta_out,
|
||||
cu_seqlens=cu_seqlens,
|
||||
chunk_indices=chunk_indices,
|
||||
# RCP_LN2 folds in the natural-log -> log2 conversion so downstream
|
||||
# exp2-based kernels reproduce exp(g). Keep this in sync with the
|
||||
# `use_exp2=True` path in `_chunk_kda_fwd_with_cumulative_g`.
|
||||
cumsum_scale=RCP_LN2,
|
||||
lower_bound=lower_bound or 0.0,
|
||||
beta=beta,
|
||||
threshold=threshold,
|
||||
T=T,
|
||||
stride_beta_batch=raw_beta.stride(0),
|
||||
stride_beta_token=raw_beta.stride(1),
|
||||
stride_beta_head=raw_beta.stride(2),
|
||||
H=H,
|
||||
S=D,
|
||||
BT=chunk_size,
|
||||
USE_LOWER_BOUND=lower_bound is not None,
|
||||
)
|
||||
return y, beta_out
|
||||
|
||||
|
||||
def _chunk_kda_fwd_with_cumulative_g(
|
||||
q: torch.Tensor,
|
||||
k: torch.Tensor,
|
||||
v: torch.Tensor,
|
||||
g: torch.Tensor,
|
||||
beta: torch.Tensor,
|
||||
scale: float,
|
||||
initial_state: torch.Tensor,
|
||||
output_final_state: bool,
|
||||
cu_seqlens: torch.Tensor | None = None,
|
||||
chunk_indices: torch.Tensor | None = None,
|
||||
chunk_size: int = FLA_CHUNK_SIZE,
|
||||
safe_gate: bool = False,
|
||||
):
|
||||
# `g` must already be chunk-local cumulatively-summed AND scaled by
|
||||
# RCP_LN2 (so the downstream exp2-based kernels reproduce exp(g)).
|
||||
# Use `chunk_kda_fwd` or `chunk_kda_with_fused_gate_fwd` instead of
|
||||
# calling this helper directly unless that invariant is upheld.
|
||||
Aqk, A = chunk_kda_fwd_intra(
|
||||
q=q,
|
||||
k=k,
|
||||
gk=g,
|
||||
beta=beta,
|
||||
scale=scale,
|
||||
cu_seqlens=cu_seqlens,
|
||||
chunk_indices=chunk_indices,
|
||||
chunk_size=chunk_size,
|
||||
safe_gate=safe_gate,
|
||||
)
|
||||
w, u, _, kg = recompute_w_u_fwd(
|
||||
k=k,
|
||||
v=v,
|
||||
beta=beta,
|
||||
A=A,
|
||||
gk=g,
|
||||
cu_seqlens=cu_seqlens,
|
||||
chunk_indices=chunk_indices,
|
||||
)
|
||||
del A
|
||||
h, v_new, final_state = chunk_gated_delta_rule_fwd_h(
|
||||
k=kg,
|
||||
w=w,
|
||||
u=u,
|
||||
gk=g,
|
||||
initial_state=initial_state,
|
||||
output_final_state=output_final_state,
|
||||
cu_seqlens=cu_seqlens,
|
||||
chunk_indices=chunk_indices,
|
||||
use_exp2=True,
|
||||
)
|
||||
del w, u, kg
|
||||
o = chunk_gla_fwd_o_gk(
|
||||
q=q,
|
||||
v=v_new,
|
||||
g=g,
|
||||
A=Aqk,
|
||||
h=h,
|
||||
o=v,
|
||||
scale=scale,
|
||||
cu_seqlens=cu_seqlens,
|
||||
chunk_indices=chunk_indices,
|
||||
chunk_size=chunk_size,
|
||||
)
|
||||
del Aqk, v_new, h
|
||||
return o, final_state
|
||||
|
||||
|
||||
def chunk_kda_fwd(
|
||||
q: torch.Tensor,
|
||||
k: torch.Tensor,
|
||||
v: torch.Tensor,
|
||||
g: torch.Tensor,
|
||||
beta: torch.Tensor,
|
||||
scale: float,
|
||||
initial_state: torch.Tensor,
|
||||
output_final_state: bool,
|
||||
cu_seqlens: torch.Tensor | None = None,
|
||||
):
|
||||
chunk_size = FLA_CHUNK_SIZE
|
||||
chunk_indices = (
|
||||
prepare_chunk_indices(cu_seqlens, chunk_size)
|
||||
if cu_seqlens is not None
|
||||
else None
|
||||
)
|
||||
g = chunk_local_cumsum(
|
||||
g,
|
||||
chunk_size=chunk_size,
|
||||
cu_seqlens=cu_seqlens,
|
||||
chunk_indices=chunk_indices,
|
||||
)
|
||||
# KDA evaluates cumulative gate decays with exp2. Convert from natural-log
|
||||
# space so exp(x) is preserved as exp2(x / ln(2)).
|
||||
g = g * RCP_LN2
|
||||
return _chunk_kda_fwd_with_cumulative_g(
|
||||
q=q,
|
||||
k=k,
|
||||
v=v,
|
||||
g=g,
|
||||
beta=beta,
|
||||
scale=scale,
|
||||
initial_state=initial_state,
|
||||
output_final_state=output_final_state,
|
||||
cu_seqlens=cu_seqlens,
|
||||
chunk_indices=chunk_indices,
|
||||
chunk_size=chunk_size,
|
||||
)
|
||||
|
||||
|
||||
def chunk_kda_with_fused_gate_fwd(
|
||||
q: torch.Tensor,
|
||||
k: torch.Tensor,
|
||||
v: torch.Tensor,
|
||||
raw_g: torch.Tensor,
|
||||
raw_beta: torch.Tensor,
|
||||
A_log: torch.Tensor,
|
||||
g_bias: torch.Tensor | None,
|
||||
scale: float,
|
||||
initial_state: torch.Tensor,
|
||||
output_final_state: bool,
|
||||
lower_bound: float | None = None,
|
||||
cu_seqlens: torch.Tensor | None = None,
|
||||
):
|
||||
chunk_size = FLA_CHUNK_SIZE
|
||||
chunk_indices = (
|
||||
prepare_chunk_indices(cu_seqlens, chunk_size)
|
||||
if cu_seqlens is not None
|
||||
else None
|
||||
)
|
||||
g, beta = fused_kda_gate_chunk_cumsum(
|
||||
raw_g,
|
||||
raw_beta=raw_beta,
|
||||
A_log=A_log,
|
||||
g_bias=g_bias,
|
||||
cu_seqlens=cu_seqlens,
|
||||
chunk_indices=chunk_indices,
|
||||
chunk_size=chunk_size,
|
||||
lower_bound=lower_bound,
|
||||
)
|
||||
return _chunk_kda_fwd_with_cumulative_g(
|
||||
q=q,
|
||||
k=k,
|
||||
v=v,
|
||||
g=g,
|
||||
beta=beta,
|
||||
scale=scale,
|
||||
initial_state=initial_state,
|
||||
output_final_state=output_final_state,
|
||||
cu_seqlens=cu_seqlens,
|
||||
chunk_indices=chunk_indices,
|
||||
chunk_size=chunk_size,
|
||||
safe_gate=lower_bound is not None,
|
||||
)
|
||||
|
||||
|
||||
def chunk_kda(
|
||||
q: torch.Tensor,
|
||||
k: torch.Tensor,
|
||||
v: torch.Tensor,
|
||||
g: torch.Tensor,
|
||||
beta: torch.Tensor,
|
||||
scale: float = None,
|
||||
initial_state: torch.Tensor = None,
|
||||
output_final_state: bool = False,
|
||||
use_qk_l2norm_in_kernel: bool = False,
|
||||
cu_seqlens: torch.Tensor | None = None,
|
||||
**kwargs,
|
||||
):
|
||||
if scale is None:
|
||||
scale = k.shape[-1] ** -0.5
|
||||
|
||||
if use_qk_l2norm_in_kernel:
|
||||
q = l2norm_fwd(q.contiguous())
|
||||
k = l2norm_fwd(k.contiguous())
|
||||
|
||||
o, final_state = chunk_kda_fwd(
|
||||
q=q,
|
||||
k=k,
|
||||
v=v.contiguous(),
|
||||
g=g.contiguous(),
|
||||
beta=beta.contiguous(),
|
||||
scale=scale,
|
||||
initial_state=initial_state.contiguous(),
|
||||
output_final_state=output_final_state,
|
||||
cu_seqlens=cu_seqlens,
|
||||
)
|
||||
return o, final_state
|
||||
|
||||
|
||||
def chunk_kda_with_fused_gate(
|
||||
q: torch.Tensor,
|
||||
k: torch.Tensor,
|
||||
v: torch.Tensor,
|
||||
raw_g: torch.Tensor,
|
||||
raw_beta: torch.Tensor,
|
||||
A_log: torch.Tensor,
|
||||
g_bias: torch.Tensor | None,
|
||||
scale: float | None = None,
|
||||
initial_state: torch.Tensor | None = None,
|
||||
output_final_state: bool = False,
|
||||
lower_bound: float | None = None,
|
||||
use_qk_l2norm_in_kernel: bool = False,
|
||||
cu_seqlens: torch.Tensor | None = None,
|
||||
**kwargs,
|
||||
):
|
||||
"""Run chunk KDA from raw gate and beta projections."""
|
||||
if scale is None:
|
||||
scale = k.shape[-1] ** -0.5
|
||||
|
||||
if use_qk_l2norm_in_kernel:
|
||||
q = l2norm_fwd(q.contiguous())
|
||||
k = l2norm_fwd(k.contiguous())
|
||||
|
||||
o, final_state = chunk_kda_with_fused_gate_fwd(
|
||||
q=q,
|
||||
k=k,
|
||||
v=v.contiguous(),
|
||||
raw_g=raw_g.contiguous(),
|
||||
raw_beta=raw_beta,
|
||||
A_log=A_log,
|
||||
g_bias=g_bias,
|
||||
scale=scale,
|
||||
initial_state=initial_state.contiguous() if initial_state is not None else None,
|
||||
output_final_state=output_final_state,
|
||||
lower_bound=lower_bound,
|
||||
cu_seqlens=cu_seqlens,
|
||||
)
|
||||
return o, final_state
|
||||
|
||||
|
||||
@triton.autotune(
|
||||
configs=[
|
||||
triton.Config({"BT": bt}, num_warps=nw, num_stages=ns)
|
||||
for bt in BT_LIST_AUTOTUNE
|
||||
for nw in NUM_WARPS_AUTOTUNE
|
||||
for ns in [2, 3]
|
||||
],
|
||||
key=["H", "D"],
|
||||
)
|
||||
@triton.jit
|
||||
def kda_gate_fwd_kernel(
|
||||
g,
|
||||
A,
|
||||
y,
|
||||
g_bias,
|
||||
lower_bound,
|
||||
beta: tl.constexpr,
|
||||
threshold: tl.constexpr,
|
||||
T,
|
||||
H,
|
||||
D: tl.constexpr,
|
||||
BT: tl.constexpr,
|
||||
BD: tl.constexpr,
|
||||
HAS_BIAS: tl.constexpr,
|
||||
USE_LOWER_BOUND: tl.constexpr,
|
||||
):
|
||||
i_t, i_h = tl.program_id(0), tl.program_id(1)
|
||||
n_t = i_t * BT
|
||||
|
||||
b_a = tl.exp(tl.load(A + i_h).to(tl.float32))
|
||||
|
||||
stride_row = H * D
|
||||
stride_col = 1
|
||||
|
||||
g_ptr = tl.make_block_ptr(
|
||||
base=g + i_h * D,
|
||||
shape=(T, D),
|
||||
strides=(stride_row, stride_col),
|
||||
offsets=(n_t, 0),
|
||||
block_shape=(BT, BD),
|
||||
order=(1, 0),
|
||||
)
|
||||
|
||||
y_ptr = tl.make_block_ptr(
|
||||
base=y + i_h * D,
|
||||
shape=(T, D),
|
||||
strides=(stride_row, stride_col),
|
||||
offsets=(n_t, 0),
|
||||
block_shape=(BT, BD),
|
||||
order=(1, 0),
|
||||
)
|
||||
|
||||
b_g = tl.load(g_ptr, boundary_check=(0, 1)).to(tl.float32)
|
||||
|
||||
if HAS_BIAS:
|
||||
n_d = tl.arange(0, BD)
|
||||
bias_mask = n_d < D
|
||||
b_bias = tl.load(g_bias + i_h * D + n_d, mask=bias_mask, other=0.0).to(
|
||||
tl.float32
|
||||
)
|
||||
b_g = b_g + b_bias[None, :]
|
||||
|
||||
if USE_LOWER_BOUND:
|
||||
b_y = lower_bound * tl.sigmoid(b_a * b_g)
|
||||
else:
|
||||
g_scaled = b_g * beta
|
||||
use_linear = g_scaled > threshold
|
||||
sp = tl.where(use_linear, b_g, (1.0 / beta) * log(1.0 + tl.exp(g_scaled)))
|
||||
b_y = -b_a * sp
|
||||
|
||||
tl.store(y_ptr, b_y.to(y.dtype.element_ty), boundary_check=(0, 1))
|
||||
|
||||
|
||||
def fused_kda_gate(
|
||||
g: torch.Tensor,
|
||||
A: torch.Tensor,
|
||||
head_k_dim: int,
|
||||
g_bias: torch.Tensor | None = None,
|
||||
beta: float = 1.0,
|
||||
threshold: float = 20.0,
|
||||
lower_bound: float | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Forward pass for KDA gate:
|
||||
input g: [..., H*D]
|
||||
param A: [H] or [1, 1, H, 1]
|
||||
beta: softplus beta parameter
|
||||
threshold: softplus threshold parameter
|
||||
return : [..., H, D]
|
||||
"""
|
||||
orig_shape = g.shape[:-1]
|
||||
|
||||
g = g.view(-1, g.shape[-1])
|
||||
T = g.shape[0]
|
||||
HD = g.shape[1]
|
||||
H = A.numel()
|
||||
assert H * head_k_dim == HD
|
||||
assert g.stride() == (HD, 1)
|
||||
|
||||
y = torch.empty_like(g, dtype=torch.float32)
|
||||
|
||||
def grid(meta):
|
||||
return (cdiv(T, meta["BT"]), H)
|
||||
|
||||
kda_gate_fwd_kernel[grid](
|
||||
g,
|
||||
A,
|
||||
y,
|
||||
g_bias,
|
||||
lower_bound or 0.0,
|
||||
beta,
|
||||
threshold,
|
||||
T,
|
||||
H,
|
||||
head_k_dim,
|
||||
BD=next_power_of_2(head_k_dim),
|
||||
HAS_BIAS=g_bias is not None,
|
||||
USE_LOWER_BOUND=lower_bound is not None,
|
||||
)
|
||||
|
||||
y = y.view(*orig_shape, H, head_k_dim)
|
||||
return y
|
||||
@@ -0,0 +1,559 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# SPDX-FileCopyrightText: Songlin Yang, Yu Zhang, Zhiyuan Li
|
||||
#
|
||||
# This file contains code copied from the flash-linear-attention project.
|
||||
# The original source was licensed under the MIT license.
|
||||
# Copyright (c) 2023-2026, Songlin Yang, Yu Zhang, Zhiyuan Li
|
||||
# Forward-only adaptation of flash-linear-attention 0.5.0.
|
||||
# ruff: noqa: E501
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.third_party.flash_linear_attention.ops.index import prepare_chunk_indices
|
||||
from vllm.third_party.flash_linear_attention.ops.op import exp2, gather
|
||||
from vllm.third_party.flash_linear_attention.ops.utils import is_gather_supported
|
||||
from vllm.triton_utils import tl, triton
|
||||
|
||||
from .chunk_intra_token_parallel import chunk_kda_fwd_intra_token_parallel
|
||||
|
||||
################################################################################
|
||||
# Fused inter + solve_tril kernel: compute off-diagonal Akk and solve in one pass
|
||||
################################################################################
|
||||
|
||||
|
||||
@triton.heuristics({
|
||||
'IS_VARLEN': lambda args: args['cu_seqlens'] is not None,
|
||||
})
|
||||
@triton.autotune(
|
||||
configs=[
|
||||
triton.Config({'BK': BK}, num_warps=num_warps)
|
||||
for BK in [32, 64]
|
||||
for num_warps in [1, 2, 4]
|
||||
],
|
||||
key=["H", "HV", "K", "BC"],
|
||||
)
|
||||
@triton.jit(do_not_specialize=['T'])
|
||||
def chunk_kda_fwd_kernel_inter_solve_fused(
|
||||
q,
|
||||
k,
|
||||
g,
|
||||
beta,
|
||||
Aqk,
|
||||
Akkd,
|
||||
Akk,
|
||||
scale,
|
||||
cu_seqlens,
|
||||
chunk_indices,
|
||||
T,
|
||||
H: tl.constexpr,
|
||||
HV: tl.constexpr,
|
||||
K: tl.constexpr,
|
||||
BT: tl.constexpr,
|
||||
BC: tl.constexpr,
|
||||
BK: tl.constexpr,
|
||||
IS_VARLEN: tl.constexpr,
|
||||
USE_SAFE_GATE: tl.constexpr,
|
||||
SOLVE_TRIL_DOT_PRECISION: tl.constexpr,
|
||||
):
|
||||
"""
|
||||
Fused kernel: compute inter-subchunk Akk + solve_tril in one pass.
|
||||
Prerequisite: token_parallel has already computed diagonal Akk blocks in Akkd.
|
||||
|
||||
This kernel:
|
||||
1. Computes off-diagonal Aqk blocks -> writes to global
|
||||
2. Computes off-diagonal Akk blocks -> keeps in registers
|
||||
3. Loads diagonal Akk blocks from Akkd (fp32)
|
||||
4. Does forward substitution on diagonals
|
||||
5. Computes merged Akk_inv
|
||||
6. Writes Akk_inv to Akk
|
||||
"""
|
||||
i_t, i_bh = tl.program_id(0), tl.program_id(1)
|
||||
i_b, i_hv = i_bh // HV, i_bh % HV
|
||||
i_h = i_hv // (HV // H)
|
||||
|
||||
if IS_VARLEN:
|
||||
i_n, i_t = tl.load(chunk_indices + i_t * 2).to(tl.int32), tl.load(chunk_indices + i_t * 2 + 1).to(tl.int32)
|
||||
bos, eos = tl.load(cu_seqlens + i_n).to(tl.int32), tl.load(cu_seqlens + i_n + 1).to(tl.int32)
|
||||
T = eos - bos
|
||||
else:
|
||||
bos, eos = i_b * T, i_b * T + T
|
||||
|
||||
if i_t * BT >= T:
|
||||
return
|
||||
|
||||
i_tc0 = i_t * BT
|
||||
i_tc1 = i_t * BT + BC
|
||||
i_tc2 = i_t * BT + 2 * BC
|
||||
i_tc3 = i_t * BT + 3 * BC
|
||||
|
||||
q += (bos * H + i_h) * K
|
||||
k += (bos * H + i_h) * K
|
||||
g += (bos * HV + i_hv) * K
|
||||
Aqk += (bos * HV + i_hv) * BT
|
||||
Akk += (bos * HV + i_hv) * BT
|
||||
Akkd += (bos * HV + i_hv) * BC
|
||||
|
||||
o_i = tl.arange(0, BC)
|
||||
m_tc1 = (i_tc1 + o_i) < T
|
||||
m_tc2 = (i_tc2 + o_i) < T
|
||||
m_tc3 = (i_tc3 + o_i) < T
|
||||
|
||||
b_Aqk10 = tl.zeros([BC, BC], dtype=tl.float32)
|
||||
b_Akk10 = tl.zeros([BC, BC], dtype=tl.float32)
|
||||
|
||||
b_Aqk20 = tl.zeros([BC, BC], dtype=tl.float32)
|
||||
b_Akk20 = tl.zeros([BC, BC], dtype=tl.float32)
|
||||
b_Aqk21 = tl.zeros([BC, BC], dtype=tl.float32)
|
||||
b_Akk21 = tl.zeros([BC, BC], dtype=tl.float32)
|
||||
|
||||
b_Aqk30 = tl.zeros([BC, BC], dtype=tl.float32)
|
||||
b_Akk30 = tl.zeros([BC, BC], dtype=tl.float32)
|
||||
b_Aqk31 = tl.zeros([BC, BC], dtype=tl.float32)
|
||||
b_Akk31 = tl.zeros([BC, BC], dtype=tl.float32)
|
||||
b_Aqk32 = tl.zeros([BC, BC], dtype=tl.float32)
|
||||
b_Akk32 = tl.zeros([BC, BC], dtype=tl.float32)
|
||||
|
||||
################################################################################
|
||||
# off-diagonal blocks
|
||||
################################################################################
|
||||
for i_k in range(tl.cdiv(K, BK)):
|
||||
o_k = i_k * BK + tl.arange(0, BK)
|
||||
m_k = o_k < K
|
||||
|
||||
p_k0 = tl.make_block_ptr(k, (T, K), (H*K, 1), (i_tc0, i_k * BK), (BC, BK), (1, 0))
|
||||
p_g0 = tl.make_block_ptr(g, (T, K), (HV*K, 1), (i_tc0, i_k * BK), (BC, BK), (1, 0))
|
||||
b_k0 = tl.load(p_k0, boundary_check=(0, 1)).to(tl.float32)
|
||||
b_g0 = tl.load(p_g0, boundary_check=(0, 1)).to(tl.float32)
|
||||
|
||||
if i_tc1 < T:
|
||||
p_q1 = tl.make_block_ptr(q, (T, K), (H*K, 1), (i_tc1, i_k * BK), (BC, BK), (1, 0))
|
||||
p_k1 = tl.make_block_ptr(k, (T, K), (H*K, 1), (i_tc1, i_k * BK), (BC, BK), (1, 0))
|
||||
p_g1 = tl.make_block_ptr(g, (T, K), (HV*K, 1), (i_tc1, i_k * BK), (BC, BK), (1, 0))
|
||||
# [BC, BK]
|
||||
b_q1 = tl.load(p_q1, boundary_check=(0, 1)).to(tl.float32)
|
||||
b_k1 = tl.load(p_k1, boundary_check=(0, 1)).to(tl.float32)
|
||||
b_g1 = tl.load(p_g1, boundary_check=(0, 1)).to(tl.float32)
|
||||
# [BK]
|
||||
b_gn1 = tl.load(g + i_tc1 * HV*K + o_k, mask=m_k, other=0).to(tl.float32)
|
||||
# [BC, BK]
|
||||
b_gqn = tl.where(m_tc1[:, None], exp2(b_g1 - b_gn1[None, :]), 0)
|
||||
# [BK, BC]
|
||||
b_kgt = tl.trans(b_k0 * exp2(b_gn1[None, :] - b_g0))
|
||||
# [BC, BC]
|
||||
b_Aqk10 += tl.dot(b_q1 * b_gqn, b_kgt)
|
||||
b_Akk10 += tl.dot(b_k1 * b_gqn, b_kgt)
|
||||
|
||||
if i_tc2 < T:
|
||||
p_q2 = tl.make_block_ptr(q, (T, K), (H*K, 1), (i_tc2, i_k * BK), (BC, BK), (1, 0))
|
||||
p_k2 = tl.make_block_ptr(k, (T, K), (H*K, 1), (i_tc2, i_k * BK), (BC, BK), (1, 0))
|
||||
p_g2 = tl.make_block_ptr(g, (T, K), (HV*K, 1), (i_tc2, i_k * BK), (BC, BK), (1, 0))
|
||||
# [BC, BK]
|
||||
b_q2 = tl.load(p_q2, boundary_check=(0, 1)).to(tl.float32)
|
||||
b_k2 = tl.load(p_k2, boundary_check=(0, 1)).to(tl.float32)
|
||||
b_g2 = tl.load(p_g2, boundary_check=(0, 1)).to(tl.float32)
|
||||
# [BK]
|
||||
b_gn2 = tl.load(g + i_tc2 * HV*K + o_k, mask=m_k, other=0).to(tl.float32)
|
||||
# [BC, BK]
|
||||
b_gqn2 = tl.where(m_tc2[:, None], exp2(b_g2 - b_gn2[None, :]), 0)
|
||||
b_qg2 = b_q2 * b_gqn2
|
||||
b_kg2 = b_k2 * b_gqn2
|
||||
# [BK, BC]
|
||||
b_kgt = tl.trans(b_k0 * exp2(b_gn2[None, :] - b_g0))
|
||||
b_Aqk20 += tl.dot(b_qg2, b_kgt)
|
||||
b_Akk20 += tl.dot(b_kg2, b_kgt)
|
||||
# [BC, BC]
|
||||
b_kgt = tl.trans(b_k1 * exp2(b_gn2[None, :] - b_g1))
|
||||
# [BC, BC]
|
||||
b_Aqk21 += tl.dot(b_qg2, b_kgt)
|
||||
b_Akk21 += tl.dot(b_kg2, b_kgt)
|
||||
|
||||
if i_tc3 < T:
|
||||
p_q3 = tl.make_block_ptr(q, (T, K), (H*K, 1), (i_tc3, i_k * BK), (BC, BK), (1, 0))
|
||||
p_k3 = tl.make_block_ptr(k, (T, K), (H*K, 1), (i_tc3, i_k * BK), (BC, BK), (1, 0))
|
||||
p_g3 = tl.make_block_ptr(g, (T, K), (HV*K, 1), (i_tc3, i_k * BK), (BC, BK), (1, 0))
|
||||
# [BC, BK]
|
||||
b_q3 = tl.load(p_q3, boundary_check=(0, 1)).to(tl.float32)
|
||||
b_k3 = tl.load(p_k3, boundary_check=(0, 1)).to(tl.float32)
|
||||
b_g3 = tl.load(p_g3, boundary_check=(0, 1)).to(tl.float32)
|
||||
# [BK]
|
||||
b_gn3 = tl.load(g + i_tc3 * HV*K + o_k, mask=m_k, other=0).to(tl.float32)
|
||||
# [BC, BK]
|
||||
b_gqn3 = tl.where(m_tc3[:, None], exp2(b_g3 - b_gn3[None, :]), 0)
|
||||
b_qg3 = b_q3 * b_gqn3
|
||||
b_kg3 = b_k3 * b_gqn3
|
||||
# [BK, BC]
|
||||
b_kgt = tl.trans(b_k0 * exp2(b_gn3[None, :] - b_g0))
|
||||
# [BC, BC]
|
||||
b_Aqk30 += tl.dot(b_qg3, b_kgt)
|
||||
b_Akk30 += tl.dot(b_kg3, b_kgt)
|
||||
# [BK, BC]
|
||||
b_kgt = tl.trans(b_k1 * exp2(b_gn3[None, :] - b_g1))
|
||||
# [BC, BC]
|
||||
b_Aqk31 += tl.dot(b_qg3, b_kgt)
|
||||
b_Akk31 += tl.dot(b_kg3, b_kgt)
|
||||
# [BK, BC]
|
||||
b_kgt = tl.trans(b_k2 * exp2(b_gn3[None, :] - b_g2))
|
||||
# [BC, BC]
|
||||
b_Aqk32 += tl.dot(b_qg3, b_kgt)
|
||||
b_Akk32 += tl.dot(b_kg3, b_kgt)
|
||||
|
||||
################################################################################
|
||||
# save off-diagonal Aqk blocks and prepare Akk
|
||||
################################################################################
|
||||
if i_tc1 < T:
|
||||
p_Aqk10 = tl.make_block_ptr(Aqk, (T, BT), (HV*BT, 1), (i_tc1, 0), (BC, BC), (1, 0))
|
||||
tl.store(p_Aqk10, (b_Aqk10 * scale).to(Aqk.dtype.element_ty), boundary_check=(0, 1))
|
||||
|
||||
p_b1 = tl.make_block_ptr(beta + bos * HV + i_hv, (T,), (HV,), (i_tc1,), (BC,), (0,))
|
||||
b_b1 = tl.load(p_b1, boundary_check=(0,)).to(tl.float32)
|
||||
b_Akk10 = b_Akk10 * b_b1[:, None]
|
||||
if i_tc2 < T:
|
||||
p_Aqk20 = tl.make_block_ptr(Aqk, (T, BT), (HV*BT, 1), (i_tc2, 0), (BC, BC), (1, 0))
|
||||
p_Aqk21 = tl.make_block_ptr(Aqk, (T, BT), (HV*BT, 1), (i_tc2, BC), (BC, BC), (1, 0))
|
||||
tl.store(p_Aqk20, (b_Aqk20 * scale).to(Aqk.dtype.element_ty), boundary_check=(0, 1))
|
||||
tl.store(p_Aqk21, (b_Aqk21 * scale).to(Aqk.dtype.element_ty), boundary_check=(0, 1))
|
||||
|
||||
p_b2 = tl.make_block_ptr(beta + bos * HV + i_hv, (T,), (HV,), (i_tc2,), (BC,), (0,))
|
||||
b_b2 = tl.load(p_b2, boundary_check=(0,)).to(tl.float32)
|
||||
b_Akk20 = b_Akk20 * b_b2[:, None]
|
||||
b_Akk21 = b_Akk21 * b_b2[:, None]
|
||||
if i_tc3 < T:
|
||||
p_Aqk30 = tl.make_block_ptr(Aqk, (T, BT), (HV*BT, 1), (i_tc3, 0), (BC, BC), (1, 0))
|
||||
p_Aqk31 = tl.make_block_ptr(Aqk, (T, BT), (HV*BT, 1), (i_tc3, BC), (BC, BC), (1, 0))
|
||||
p_Aqk32 = tl.make_block_ptr(Aqk, (T, BT), (HV*BT, 1), (i_tc3, 2*BC), (BC, BC), (1, 0))
|
||||
tl.store(p_Aqk30, (b_Aqk30 * scale).to(Aqk.dtype.element_ty), boundary_check=(0, 1))
|
||||
tl.store(p_Aqk31, (b_Aqk31 * scale).to(Aqk.dtype.element_ty), boundary_check=(0, 1))
|
||||
tl.store(p_Aqk32, (b_Aqk32 * scale).to(Aqk.dtype.element_ty), boundary_check=(0, 1))
|
||||
|
||||
p_b3 = tl.make_block_ptr(beta + bos * HV + i_hv, (T,), (HV,), (i_tc3,), (BC,), (0,))
|
||||
b_b3 = tl.load(p_b3, boundary_check=(0,)).to(tl.float32)
|
||||
b_Akk30 = b_Akk30 * b_b3[:, None]
|
||||
b_Akk31 = b_Akk31 * b_b3[:, None]
|
||||
b_Akk32 = b_Akk32 * b_b3[:, None]
|
||||
|
||||
p_Akk00 = tl.make_block_ptr(Akkd, (T, BC), (HV*BC, 1), (i_tc0, 0), (BC, BC), (1, 0))
|
||||
p_Akk11 = tl.make_block_ptr(Akkd, (T, BC), (HV*BC, 1), (i_tc1, 0), (BC, BC), (1, 0))
|
||||
p_Akk22 = tl.make_block_ptr(Akkd, (T, BC), (HV*BC, 1), (i_tc2, 0), (BC, BC), (1, 0))
|
||||
p_Akk33 = tl.make_block_ptr(Akkd, (T, BC), (HV*BC, 1), (i_tc3, 0), (BC, BC), (1, 0))
|
||||
b_Ai00 = tl.load(p_Akk00, boundary_check=(0, 1)).to(tl.float32)
|
||||
b_Ai11 = tl.load(p_Akk11, boundary_check=(0, 1)).to(tl.float32)
|
||||
b_Ai22 = tl.load(p_Akk22, boundary_check=(0, 1)).to(tl.float32)
|
||||
b_Ai33 = tl.load(p_Akk33, boundary_check=(0, 1)).to(tl.float32)
|
||||
|
||||
################################################################################
|
||||
# forward substitution on diagonals
|
||||
################################################################################
|
||||
|
||||
if not USE_SAFE_GATE:
|
||||
m_A = o_i[:, None] > o_i[None, :]
|
||||
m_I = o_i[:, None] == o_i[None, :]
|
||||
|
||||
b_Ai00 = -tl.where(m_A, b_Ai00, 0)
|
||||
b_Ai11 = -tl.where(m_A, b_Ai11, 0)
|
||||
b_Ai22 = -tl.where(m_A, b_Ai22, 0)
|
||||
b_Ai33 = -tl.where(m_A, b_Ai33, 0)
|
||||
|
||||
for i in range(2, min(BC, T - i_tc0)):
|
||||
b_a00 = -tl.load(Akkd + (i_tc0 + i) * HV*BC + o_i)
|
||||
b_a00 = tl.where(o_i < i, b_a00, 0.)
|
||||
b_a00 += tl.sum(b_a00[:, None] * b_Ai00, 0)
|
||||
b_Ai00 = tl.where((o_i == i)[:, None], b_a00, b_Ai00)
|
||||
for i in range(BC + 2, min(2*BC, T - i_tc0)):
|
||||
b_a11 = -tl.load(Akkd + (i_tc0 + i) * HV*BC + o_i)
|
||||
b_a11 = tl.where(o_i < i - BC, b_a11, 0.)
|
||||
b_a11 += tl.sum(b_a11[:, None] * b_Ai11, 0)
|
||||
b_Ai11 = tl.where((o_i == i - BC)[:, None], b_a11, b_Ai11)
|
||||
for i in range(2*BC + 2, min(3*BC, T - i_tc0)):
|
||||
b_a22 = -tl.load(Akkd + (i_tc0 + i) * HV*BC + o_i)
|
||||
b_a22 = tl.where(o_i < i - 2*BC, b_a22, 0.)
|
||||
b_a22 += tl.sum(b_a22[:, None] * b_Ai22, 0)
|
||||
b_Ai22 = tl.where((o_i == i - 2*BC)[:, None], b_a22, b_Ai22)
|
||||
for i in range(3*BC + 2, min(4*BC, T - i_tc0)):
|
||||
b_a33 = -tl.load(Akkd + (i_tc0 + i) * HV*BC + o_i)
|
||||
b_a33 = tl.where(o_i < i - 3*BC, b_a33, 0.)
|
||||
b_a33 += tl.sum(b_a33[:, None] * b_Ai33, 0)
|
||||
b_Ai33 = tl.where((o_i == i - 3*BC)[:, None], b_a33, b_Ai33)
|
||||
|
||||
b_Ai00 += m_I
|
||||
b_Ai11 += m_I
|
||||
b_Ai22 += m_I
|
||||
b_Ai33 += m_I
|
||||
|
||||
################################################################################
|
||||
# compute merged inverse using off-diagonals
|
||||
################################################################################
|
||||
|
||||
# we used tf32 to maintain matrix inverse's precision whenever possible.
|
||||
b_Ai10 = -tl.dot(
|
||||
tl.dot(b_Ai11, b_Akk10, input_precision=SOLVE_TRIL_DOT_PRECISION),
|
||||
b_Ai00,
|
||||
input_precision=SOLVE_TRIL_DOT_PRECISION
|
||||
)
|
||||
b_Ai21 = -tl.dot(
|
||||
tl.dot(b_Ai22, b_Akk21, input_precision=SOLVE_TRIL_DOT_PRECISION),
|
||||
b_Ai11,
|
||||
input_precision=SOLVE_TRIL_DOT_PRECISION
|
||||
)
|
||||
b_Ai32 = -tl.dot(
|
||||
tl.dot(b_Ai33, b_Akk32, input_precision=SOLVE_TRIL_DOT_PRECISION),
|
||||
b_Ai22,
|
||||
input_precision=SOLVE_TRIL_DOT_PRECISION
|
||||
)
|
||||
|
||||
b_Ai20 = -tl.dot(
|
||||
b_Ai22,
|
||||
tl.dot(b_Akk20, b_Ai00, input_precision=SOLVE_TRIL_DOT_PRECISION) +
|
||||
tl.dot(b_Akk21, b_Ai10, input_precision=SOLVE_TRIL_DOT_PRECISION),
|
||||
input_precision=SOLVE_TRIL_DOT_PRECISION
|
||||
)
|
||||
b_Ai31 = -tl.dot(
|
||||
b_Ai33,
|
||||
tl.dot(b_Akk31, b_Ai11, input_precision=SOLVE_TRIL_DOT_PRECISION) +
|
||||
tl.dot(b_Akk32, b_Ai21, input_precision=SOLVE_TRIL_DOT_PRECISION),
|
||||
input_precision=SOLVE_TRIL_DOT_PRECISION
|
||||
)
|
||||
b_Ai30 = -tl.dot(
|
||||
b_Ai33,
|
||||
tl.dot(b_Akk30, b_Ai00, input_precision=SOLVE_TRIL_DOT_PRECISION) +
|
||||
tl.dot(b_Akk31, b_Ai10, input_precision=SOLVE_TRIL_DOT_PRECISION) +
|
||||
tl.dot(b_Akk32, b_Ai20, input_precision=SOLVE_TRIL_DOT_PRECISION),
|
||||
input_precision=SOLVE_TRIL_DOT_PRECISION
|
||||
)
|
||||
|
||||
################################################################################
|
||||
# store full Akk_inv to Akk
|
||||
################################################################################
|
||||
|
||||
p_Akk00 = tl.make_block_ptr(Akk, (T, BT), (HV*BT, 1), (i_tc0, 0), (BC, BC), (1, 0))
|
||||
p_Akk10 = tl.make_block_ptr(Akk, (T, BT), (HV*BT, 1), (i_tc1, 0), (BC, BC), (1, 0))
|
||||
p_Akk11 = tl.make_block_ptr(Akk, (T, BT), (HV*BT, 1), (i_tc1, BC), (BC, BC), (1, 0))
|
||||
p_Akk20 = tl.make_block_ptr(Akk, (T, BT), (HV*BT, 1), (i_tc2, 0), (BC, BC), (1, 0))
|
||||
p_Akk21 = tl.make_block_ptr(Akk, (T, BT), (HV*BT, 1), (i_tc2, BC), (BC, BC), (1, 0))
|
||||
p_Akk22 = tl.make_block_ptr(Akk, (T, BT), (HV*BT, 1), (i_tc2, 2*BC), (BC, BC), (1, 0))
|
||||
p_Akk30 = tl.make_block_ptr(Akk, (T, BT), (HV*BT, 1), (i_tc3, 0), (BC, BC), (1, 0))
|
||||
p_Akk31 = tl.make_block_ptr(Akk, (T, BT), (HV*BT, 1), (i_tc3, BC), (BC, BC), (1, 0))
|
||||
p_Akk32 = tl.make_block_ptr(Akk, (T, BT), (HV*BT, 1), (i_tc3, 2*BC), (BC, BC), (1, 0))
|
||||
p_Akk33 = tl.make_block_ptr(Akk, (T, BT), (HV*BT, 1), (i_tc3, 3*BC), (BC, BC), (1, 0))
|
||||
|
||||
tl.store(p_Akk00, b_Ai00.to(Akk.dtype.element_ty), boundary_check=(0, 1))
|
||||
tl.store(p_Akk10, b_Ai10.to(Akk.dtype.element_ty), boundary_check=(0, 1))
|
||||
tl.store(p_Akk11, b_Ai11.to(Akk.dtype.element_ty), boundary_check=(0, 1))
|
||||
tl.store(p_Akk20, b_Ai20.to(Akk.dtype.element_ty), boundary_check=(0, 1))
|
||||
tl.store(p_Akk21, b_Ai21.to(Akk.dtype.element_ty), boundary_check=(0, 1))
|
||||
tl.store(p_Akk22, b_Ai22.to(Akk.dtype.element_ty), boundary_check=(0, 1))
|
||||
tl.store(p_Akk30, b_Ai30.to(Akk.dtype.element_ty), boundary_check=(0, 1))
|
||||
tl.store(p_Akk31, b_Ai31.to(Akk.dtype.element_ty), boundary_check=(0, 1))
|
||||
tl.store(p_Akk32, b_Ai32.to(Akk.dtype.element_ty), boundary_check=(0, 1))
|
||||
tl.store(p_Akk33, b_Ai33.to(Akk.dtype.element_ty), boundary_check=(0, 1))
|
||||
|
||||
|
||||
@triton.heuristics({
|
||||
'IS_VARLEN': lambda args: args['cu_seqlens'] is not None,
|
||||
})
|
||||
@triton.autotune(
|
||||
configs=[
|
||||
triton.Config({}, num_warps=num_warps, num_stages=num_stages)
|
||||
for num_warps in [1, 2, 4, 8]
|
||||
for num_stages in [2, 3, 4]
|
||||
],
|
||||
key=['BK', 'NC', 'BT', 'HV'],
|
||||
)
|
||||
@triton.jit(do_not_specialize=['B', 'T'])
|
||||
def chunk_kda_fwd_kernel_intra_sub_chunk(
|
||||
q,
|
||||
k,
|
||||
g,
|
||||
beta,
|
||||
Aqk,
|
||||
Akk,
|
||||
scale,
|
||||
cu_seqlens,
|
||||
chunk_indices,
|
||||
T,
|
||||
H: tl.constexpr,
|
||||
HV: tl.constexpr,
|
||||
K: tl.constexpr,
|
||||
BT: tl.constexpr,
|
||||
BC: tl.constexpr,
|
||||
BK: tl.constexpr,
|
||||
IS_VARLEN: tl.constexpr,
|
||||
USE_GATHER: tl.constexpr,
|
||||
):
|
||||
i_t, i_i, i_bh = tl.program_id(0), tl.program_id(1), tl.program_id(2)
|
||||
i_b, i_hv = i_bh // HV, i_bh % HV
|
||||
i_h = i_hv // (HV // H)
|
||||
|
||||
if IS_VARLEN:
|
||||
i_n, i_t = tl.load(chunk_indices + i_t * 2).to(tl.int32), tl.load(chunk_indices + i_t * 2 + 1).to(tl.int32)
|
||||
bos, eos = tl.load(cu_seqlens + i_n).to(tl.int32), tl.load(cu_seqlens + i_n + 1).to(tl.int32)
|
||||
T = eos - bos
|
||||
else:
|
||||
bos, eos = i_b * T, i_b * T + T
|
||||
|
||||
i_ti = i_t * BT + i_i * BC
|
||||
if i_ti >= T:
|
||||
return
|
||||
|
||||
o_c = i_ti + tl.arange(0, BC)
|
||||
m_c = o_c < T
|
||||
|
||||
q = q + (bos * H + i_h) * K
|
||||
k = k + (bos * H + i_h) * K
|
||||
g = g + (bos * HV + i_hv) * K
|
||||
beta = beta + bos * HV + i_hv
|
||||
Aqk = Aqk + (bos * HV + i_hv) * BT
|
||||
Akk = Akk + (bos * HV + i_hv) * BC
|
||||
|
||||
p_q = tl.make_block_ptr(q, (T, K), (H*K, 1), (i_ti, 0), (BC, BK), (1, 0))
|
||||
p_k = tl.make_block_ptr(k, (T, K), (H*K, 1), (i_ti, 0), (BC, BK), (1, 0))
|
||||
p_g = tl.make_block_ptr(g, (T, K), (HV*K, 1), (i_ti, 0), (BC, BK), (1, 0))
|
||||
|
||||
p_beta = tl.make_block_ptr(beta, (T,), (HV,), (i_ti,), (BC,), (0,))
|
||||
|
||||
b_q = tl.load(p_q, boundary_check=(0, 1))
|
||||
b_k = tl.load(p_k, boundary_check=(0, 1))
|
||||
b_g = tl.load(p_g, boundary_check=(0, 1))
|
||||
b_beta = tl.load(p_beta, boundary_check=(0,)).to(tl.float32)
|
||||
|
||||
if USE_GATHER:
|
||||
b_gn = gather(b_g, tl.full([1, BK], min(BC//2, T - i_ti - 1), dtype=tl.int16), axis=0)
|
||||
else:
|
||||
# caculate offset
|
||||
p_gn = g + (i_ti + min(BC // 2, T - i_ti - 1)) * HV*K + tl.arange(0, BK)
|
||||
b_gn = tl.load(p_gn, mask=tl.arange(0, BK) < K, other=0.0)
|
||||
b_gn = b_gn[None, :]
|
||||
|
||||
# current block, keep numerical stability by subtracting the left boundary
|
||||
# less than 85 to avoid overflow in exp2
|
||||
b_gm = (b_g - b_gn).to(tl.float32)
|
||||
|
||||
b_gq = tl.where(m_c[:, None], exp2(b_gm), 0.)
|
||||
b_gk = tl.where(m_c[:, None], exp2(-b_gm), 0.)
|
||||
|
||||
b_kgt = tl.trans(b_k * b_gk)
|
||||
|
||||
b_Aqk = tl.dot(b_q * b_gq, b_kgt) * scale
|
||||
b_Akk = tl.dot(b_k * b_gq, b_kgt) * b_beta[:, None]
|
||||
|
||||
o_i = tl.arange(0, BC)
|
||||
m_Aqk = o_i[:, None] >= o_i[None, :]
|
||||
m_Akk = o_i[:, None] > o_i[None, :]
|
||||
m_I = o_i[:, None] == o_i[None, :]
|
||||
|
||||
b_Aqk = tl.where(m_Aqk, b_Aqk, 0.0)
|
||||
b_Akk = tl.where(m_Akk, b_Akk, 0.0)
|
||||
|
||||
p_Aqk = tl.make_block_ptr(Aqk, (T, BT), (HV*BT, 1), (i_ti, i_i * BC), (BC, BC), (1, 0))
|
||||
p_Akk = tl.make_block_ptr(Akk, (T, BC), (HV*BC, 1), (i_ti, 0), (BC, BC), (1, 0))
|
||||
tl.store(p_Aqk, b_Aqk.to(Aqk.dtype.element_ty), boundary_check=(0, 1))
|
||||
tl.store(p_Akk, b_Akk.to(Akk.dtype.element_ty), boundary_check=(0, 1))
|
||||
|
||||
tl.debug_barrier()
|
||||
|
||||
################################################################################
|
||||
# forward substitution
|
||||
################################################################################
|
||||
|
||||
b_Ai = -b_Akk
|
||||
for i in range(2, min(BC, T - i_ti)):
|
||||
b_a = -tl.load(Akk + (i_ti + i) * HV*BC + o_i)
|
||||
b_a = tl.where(o_i < i, b_a, 0.)
|
||||
b_a += tl.sum(b_a[:, None] * b_Ai, 0)
|
||||
b_Ai = tl.where((o_i == i)[:, None], b_a, b_Ai)
|
||||
b_Ai += m_I
|
||||
tl.store(p_Akk, b_Ai.to(Akk.dtype.element_ty), boundary_check=(0, 1))
|
||||
|
||||
|
||||
def chunk_kda_fwd_intra(
|
||||
q: torch.Tensor,
|
||||
k: torch.Tensor,
|
||||
gk: torch.Tensor | None = None,
|
||||
beta: torch.Tensor | None = None,
|
||||
scale: float | None = None,
|
||||
cu_seqlens: torch.LongTensor | None = None,
|
||||
chunk_size: int = 64,
|
||||
chunk_indices: torch.LongTensor | None = None,
|
||||
safe_gate: bool = False,
|
||||
):
|
||||
B, T, H, K, HV = *k.shape, gk.shape[2]
|
||||
BT = chunk_size
|
||||
BC = 16
|
||||
if chunk_indices is None and cu_seqlens is not None:
|
||||
chunk_indices = prepare_chunk_indices(cu_seqlens, BT)
|
||||
NT = triton.cdiv(T, BT) if cu_seqlens is None else len(chunk_indices)
|
||||
NC = triton.cdiv(BT, BC)
|
||||
|
||||
Aqk = torch.empty(B, T, HV, BT, device=k.device, dtype=k.dtype)
|
||||
# Akk must be zero-initialized - kernel only writes lower triangular
|
||||
Akk = torch.zeros(B, T, HV, BT, device=k.device, dtype=k.dtype)
|
||||
# Separate fp32 buffer for diagonal 16x16 blocks (for precision in solve_tril)
|
||||
Akkd = torch.empty(B, T, HV, BC, device=k.device, dtype=torch.float32)
|
||||
|
||||
# Compute diagonal blocks into Akkd in fp32.
|
||||
if safe_gate:
|
||||
grid = (NT, NC, B * HV)
|
||||
BK = triton.next_power_of_2(K)
|
||||
chunk_kda_fwd_kernel_intra_sub_chunk[grid](
|
||||
q=q,
|
||||
k=k,
|
||||
g=gk,
|
||||
beta=beta,
|
||||
Aqk=Aqk,
|
||||
Akk=Akkd,
|
||||
scale=scale,
|
||||
cu_seqlens=cu_seqlens,
|
||||
chunk_indices=chunk_indices,
|
||||
T=T,
|
||||
H=H,
|
||||
HV=HV,
|
||||
K=K,
|
||||
BT=BT,
|
||||
BC=BC,
|
||||
BK=BK,
|
||||
USE_GATHER=is_gather_supported,
|
||||
)
|
||||
else:
|
||||
Aqk, Akkd = chunk_kda_fwd_intra_token_parallel(
|
||||
q=q,
|
||||
k=k,
|
||||
gk=gk,
|
||||
beta=beta,
|
||||
Aqk=Aqk,
|
||||
Akk=Akkd,
|
||||
scale=scale,
|
||||
cu_seqlens=cu_seqlens,
|
||||
chunk_size=BT,
|
||||
sub_chunk_size=BC,
|
||||
)
|
||||
|
||||
# Step 2: Fused inter + solve_tril (works for both fixed-len and varlen)
|
||||
solve_tril_dot_precision = (
|
||||
"tf32"
|
||||
if current_platform.is_cuda()
|
||||
and current_platform.has_device_capability(80)
|
||||
else "ieee"
|
||||
)
|
||||
grid = (NT, B * HV)
|
||||
chunk_kda_fwd_kernel_inter_solve_fused[grid](
|
||||
q=q,
|
||||
k=k,
|
||||
g=gk,
|
||||
beta=beta,
|
||||
Aqk=Aqk,
|
||||
Akkd=Akkd,
|
||||
Akk=Akk,
|
||||
scale=scale,
|
||||
cu_seqlens=cu_seqlens,
|
||||
chunk_indices=chunk_indices,
|
||||
T=T,
|
||||
H=H,
|
||||
HV=HV,
|
||||
K=K,
|
||||
BT=BT,
|
||||
BC=BC,
|
||||
USE_SAFE_GATE=safe_gate,
|
||||
SOLVE_TRIL_DOT_PRECISION=solve_tril_dot_precision,
|
||||
)
|
||||
return Aqk, Akk
|
||||
+178
@@ -0,0 +1,178 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# SPDX-FileCopyrightText: Songlin Yang, Yu Zhang, Zhiyuan Li
|
||||
#
|
||||
# This file contains code copied from the flash-linear-attention project.
|
||||
# The original source was licensed under the MIT license.
|
||||
# Copyright (c) 2023-2026, Songlin Yang, Yu Zhang, Zhiyuan Li
|
||||
# Forward-only adaptation of flash-linear-attention 0.5.0.
|
||||
# ruff: noqa: E501
|
||||
|
||||
# Token-parallel implementation of KDA intra chunk kernel
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.third_party.flash_linear_attention.ops.op import exp2
|
||||
from vllm.triton_utils import tl, triton
|
||||
|
||||
|
||||
@triton.heuristics({
|
||||
'IS_VARLEN': lambda args: args['cu_seqlens'] is not None,
|
||||
})
|
||||
@triton.autotune(
|
||||
configs=[
|
||||
triton.Config({'BH': BH}, num_warps=num_warps)
|
||||
for BH in [1, 2, 4, 8]
|
||||
for num_warps in [1, 2, 4, 8]
|
||||
],
|
||||
key=["K", "H", "HV"],
|
||||
)
|
||||
@triton.jit(do_not_specialize=['T', 'N'])
|
||||
def chunk_kda_fwd_kernel_intra_token_parallel(
|
||||
q,
|
||||
k,
|
||||
g,
|
||||
beta,
|
||||
Aqk,
|
||||
Akk,
|
||||
scale,
|
||||
cu_seqlens,
|
||||
N,
|
||||
T,
|
||||
H: tl.constexpr,
|
||||
HV: tl.constexpr,
|
||||
K: tl.constexpr,
|
||||
BT: tl.constexpr,
|
||||
BC: tl.constexpr,
|
||||
BH: tl.constexpr,
|
||||
IS_VARLEN: tl.constexpr,
|
||||
):
|
||||
i_tg, i_hg = tl.program_id(0), tl.program_id(1)
|
||||
|
||||
if IS_VARLEN:
|
||||
i_n = 0
|
||||
left, right = 0, N
|
||||
|
||||
# Unrolled binary search (max B=2^32)
|
||||
# We can limit iterations based on expected max batch size if needed
|
||||
# 20 iterations covers B=1M, usually enough
|
||||
for _ in range(20):
|
||||
if left < right:
|
||||
mid = (left + right) // 2
|
||||
if i_tg < tl.load(cu_seqlens + mid + 1).to(tl.int32):
|
||||
right = mid
|
||||
else:
|
||||
left = mid + 1
|
||||
i_n = left
|
||||
|
||||
bos, eos = tl.load(cu_seqlens + i_n).to(tl.int32), tl.load(cu_seqlens + i_n + 1).to(tl.int32)
|
||||
T = eos - bos
|
||||
i_t = i_tg - bos
|
||||
else:
|
||||
bos = (i_tg // T) * T
|
||||
i_t = i_tg % T
|
||||
|
||||
if i_t >= T:
|
||||
return
|
||||
|
||||
i_c = i_t // BT
|
||||
i_s = (i_t % BT) // BC
|
||||
i_tc = i_c * BT
|
||||
i_ts = i_tc + i_s * BC
|
||||
|
||||
G: tl.constexpr = HV // H
|
||||
|
||||
q += bos * H*K
|
||||
k += bos * H*K
|
||||
g += bos * HV*K
|
||||
Aqk += bos * HV*BT
|
||||
Akk += bos * HV*BC
|
||||
beta += bos * HV
|
||||
|
||||
BK: tl.constexpr = triton.next_power_of_2(K)
|
||||
o_hv = i_hg * BH + tl.arange(0, BH)
|
||||
o_h = o_hv // G
|
||||
o_k = tl.arange(0, BK)
|
||||
m_hv = o_hv < HV
|
||||
m_k = o_k < K
|
||||
m_hk = m_hv[:, None] & m_k[None, :]
|
||||
|
||||
# q/k: [B, T, H, K], manual load via mapped qk head index
|
||||
p_qk = o_h[:, None] * K + o_k[None, :]
|
||||
b_q = tl.load(q + i_t * H * K + p_qk, mask=m_hk, other=0).to(tl.float32)
|
||||
b_k = tl.load(k + i_t * H * K + p_qk, mask=m_hk, other=0).to(tl.float32)
|
||||
|
||||
# g: [B, T, HV, K], beta: [B, T, HV]
|
||||
p_g = tl.make_block_ptr(g + i_t * HV * K, (HV, K), (K, 1), (i_hg * BH, 0), (BH, BK), (1, 0))
|
||||
p_beta = tl.make_block_ptr(beta + i_t * HV, (HV,), (1,), (i_hg * BH,), (BH,), (0,))
|
||||
b_g = tl.load(p_g, boundary_check=(0, 1)).to(tl.float32)
|
||||
b_beta = tl.load(p_beta, boundary_check=(0,)).to(tl.float32)
|
||||
b_k *= b_beta[:, None]
|
||||
|
||||
for j in range(i_ts, min(i_t + 1, min(T, i_ts + BC))):
|
||||
b_kj = tl.load(k + j * H * K + p_qk, mask=m_hk, other=0).to(tl.float32)
|
||||
p_gj = tl.make_block_ptr(g + j * HV * K, (HV, K), (K, 1), (i_hg * BH, 0), (BH, BK), (1, 0))
|
||||
b_gj = tl.load(p_gj, boundary_check=(0, 1)).to(tl.float32)
|
||||
|
||||
b_kgj = tl.where(m_k[None, :], b_kj * exp2(b_g - b_gj), 0.0)
|
||||
b_Aqk = tl.sum(b_q * b_kgj, axis=1) * scale
|
||||
b_Akk = tl.sum(b_k * b_kgj, axis=1) * tl.where(j < i_t, 1.0, 0.0)
|
||||
|
||||
tl.store(Aqk + i_t * HV * BT + o_hv * BT + j % BT, b_Aqk.to(Aqk.dtype.element_ty), mask=m_hv)
|
||||
tl.store(Akk + i_t * HV * BC + o_hv * BC + j - i_ts, b_Akk.to(Akk.dtype.element_ty), mask=m_hv)
|
||||
|
||||
|
||||
def chunk_kda_fwd_intra_token_parallel(
|
||||
q: torch.Tensor,
|
||||
k: torch.Tensor,
|
||||
gk: torch.Tensor,
|
||||
beta: torch.Tensor,
|
||||
Aqk: torch.Tensor,
|
||||
Akk: torch.Tensor,
|
||||
scale: float,
|
||||
cu_seqlens: torch.LongTensor | None = None,
|
||||
chunk_size: int = 64,
|
||||
sub_chunk_size: int = 16,
|
||||
) -> None:
|
||||
"""
|
||||
Token-parallel implementation: each token gets its own thread block.
|
||||
Supports both fixed-length and variable-length sequences.
|
||||
Reduces wasted computation on padding.
|
||||
|
||||
Writes directly to Aqk and Akk tensors (in-place).
|
||||
|
||||
Args:
|
||||
q: [B, T, H, K]
|
||||
k: [B, T, H, K]
|
||||
gk: [B, T, HV, K] cumsum of gates (HV >= H for GVA)
|
||||
beta: [B, T, HV]
|
||||
Aqk: [B, T, HV, BT] output tensor to write to
|
||||
Akk: [B, T, HV, BC] output tensor for diagonal blocks (fp32)
|
||||
scale: attention scale
|
||||
chunk_size: BT (default 64)
|
||||
sub_chunk_size: BC (default 16)
|
||||
"""
|
||||
B, T, H, K, HV = *q.shape, gk.shape[2]
|
||||
N = len(cu_seqlens) - 1 if cu_seqlens is not None else B
|
||||
BT = chunk_size
|
||||
BC = sub_chunk_size
|
||||
|
||||
def grid(meta): return (B * T, triton.cdiv(HV, meta['BH']))
|
||||
chunk_kda_fwd_kernel_intra_token_parallel[grid](
|
||||
q=q,
|
||||
k=k,
|
||||
g=gk,
|
||||
beta=beta,
|
||||
Aqk=Aqk,
|
||||
Akk=Akk,
|
||||
scale=scale,
|
||||
cu_seqlens=cu_seqlens,
|
||||
N=N,
|
||||
T=T,
|
||||
H=H,
|
||||
HV=HV,
|
||||
K=K,
|
||||
BT=BT,
|
||||
BC=BC,
|
||||
)
|
||||
return Aqk, Akk
|
||||
@@ -0,0 +1,671 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# SPDX-FileCopyrightText: Songlin Yang, Yu Zhang, Zhiyuan Li
|
||||
#
|
||||
# This file contains code adapted from the flash-linear-attention project.
|
||||
# The original source was licensed under the MIT license.
|
||||
# Copyright (c) 2023-2026, Songlin Yang, Yu Zhang, Zhiyuan Li
|
||||
# ruff: noqa: E501
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.third_party.flash_linear_attention.ops.op import exp, log
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.utils.math_utils import cdiv, next_power_of_2
|
||||
|
||||
|
||||
@triton.heuristics(
|
||||
{
|
||||
"HAS_DT_BIAS": lambda args: args["dt_bias"] is not None,
|
||||
"USE_LOWER_BOUND": lambda args: args["lower_bound"] is not None,
|
||||
}
|
||||
)
|
||||
@triton.jit
|
||||
def _kda_gate_beta_fwd_kernel(
|
||||
raw_g,
|
||||
raw_beta,
|
||||
A_log,
|
||||
dt_bias,
|
||||
gate,
|
||||
beta_out,
|
||||
lower_bound,
|
||||
softplus_beta: tl.constexpr,
|
||||
softplus_threshold: tl.constexpr,
|
||||
T,
|
||||
stride_g_token: tl.constexpr,
|
||||
stride_beta_token: tl.constexpr,
|
||||
H: tl.constexpr,
|
||||
D: tl.constexpr,
|
||||
BT: tl.constexpr,
|
||||
BD: tl.constexpr,
|
||||
HAS_DT_BIAS: tl.constexpr,
|
||||
USE_LOWER_BOUND: tl.constexpr,
|
||||
launch_pdl: tl.constexpr,
|
||||
):
|
||||
if launch_pdl:
|
||||
tl.extra.cuda.gdc_wait()
|
||||
tl.extra.cuda.gdc_launch_dependents()
|
||||
|
||||
i_t, i_h = tl.program_id(0), tl.program_id(1)
|
||||
o_t = i_t * BT + tl.arange(0, BT)
|
||||
o_d = tl.arange(0, BD)
|
||||
m_t = o_t < T
|
||||
m_d = o_d < D
|
||||
|
||||
p_g = raw_g + o_t[:, None] * stride_g_token + i_h * D + o_d[None, :]
|
||||
b_g = tl.load(p_g, mask=m_t[:, None] & m_d[None, :], other=0.0).to(tl.float32)
|
||||
if HAS_DT_BIAS:
|
||||
b_bias = tl.load(
|
||||
dt_bias + i_h * D + o_d,
|
||||
mask=m_d,
|
||||
other=0.0,
|
||||
).to(tl.float32)
|
||||
b_g += b_bias[None, :]
|
||||
|
||||
b_a = exp(tl.load(A_log + i_h).to(tl.float32))
|
||||
if USE_LOWER_BOUND:
|
||||
b_gate = lower_bound * tl.sigmoid(b_a * b_g)
|
||||
else:
|
||||
b_scaled = b_g * softplus_beta
|
||||
b_softplus = tl.where(
|
||||
b_scaled > softplus_threshold,
|
||||
b_g,
|
||||
log(1.0 + tl.exp(b_scaled)) / softplus_beta,
|
||||
)
|
||||
b_gate = -b_a * b_softplus
|
||||
|
||||
p_gate = gate + (o_t[:, None] * H + i_h) * D + o_d[None, :]
|
||||
tl.store(
|
||||
p_gate,
|
||||
b_gate,
|
||||
mask=m_t[:, None] & m_d[None, :],
|
||||
)
|
||||
|
||||
b_beta = tl.load(
|
||||
raw_beta + o_t * stride_beta_token + i_h,
|
||||
mask=m_t,
|
||||
other=0.0,
|
||||
).to(tl.float32)
|
||||
tl.store(beta_out + o_t * H + i_h, tl.sigmoid(b_beta), mask=m_t)
|
||||
|
||||
|
||||
def _fused_kda_gate_beta(
|
||||
raw_g: torch.Tensor,
|
||||
raw_beta: torch.Tensor,
|
||||
A_log: torch.Tensor,
|
||||
dt_bias: torch.Tensor | None,
|
||||
lower_bound: float | None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
B, T, H, D = raw_g.shape
|
||||
assert B == 1
|
||||
assert raw_beta.shape == (B, T, H)
|
||||
assert raw_g.stride()[2:] == (D, 1)
|
||||
assert raw_beta.stride(2) == 1
|
||||
gate = torch.empty((B, T, H, D), dtype=torch.float32, device=raw_g.device)
|
||||
beta = torch.empty((B, T, H), dtype=torch.float32, device=raw_beta.device)
|
||||
|
||||
BT = 16
|
||||
_kda_gate_beta_fwd_kernel[(cdiv(T, BT), H)](
|
||||
raw_g=raw_g,
|
||||
raw_beta=raw_beta,
|
||||
A_log=A_log,
|
||||
dt_bias=dt_bias,
|
||||
gate=gate,
|
||||
beta_out=beta,
|
||||
lower_bound=lower_bound,
|
||||
softplus_beta=1.0,
|
||||
softplus_threshold=20.0,
|
||||
T=T,
|
||||
stride_g_token=raw_g.stride(1),
|
||||
stride_beta_token=raw_beta.stride(1),
|
||||
H=H,
|
||||
D=D,
|
||||
BT=BT,
|
||||
BD=next_power_of_2(D),
|
||||
num_warps=4,
|
||||
launch_pdl=current_platform.is_arch_support_pdl(),
|
||||
)
|
||||
return gate, beta
|
||||
|
||||
|
||||
@triton.heuristics(
|
||||
{
|
||||
"IS_SPEC_DECODING": lambda args: args["num_accepted_tokens"] is not None,
|
||||
"HAS_DT_BIAS": lambda args: args["dt_bias"] is not None,
|
||||
"USE_LOWER_BOUND": lambda args: args["lower_bound"] is not None,
|
||||
}
|
||||
)
|
||||
@triton.jit(do_not_specialize=["N", "T", "stride_beta_token"])
|
||||
def fused_recurrent_kda_fwd_kernel(
|
||||
q,
|
||||
k,
|
||||
v,
|
||||
g,
|
||||
beta,
|
||||
A_log,
|
||||
dt_bias,
|
||||
out,
|
||||
state,
|
||||
cu_seqlens,
|
||||
state_indices,
|
||||
num_accepted_tokens,
|
||||
lower_bound,
|
||||
scale: tl.constexpr,
|
||||
N: tl.int64,
|
||||
T: tl.int64,
|
||||
H: tl.constexpr,
|
||||
K: tl.constexpr,
|
||||
V: tl.constexpr,
|
||||
BK: tl.constexpr,
|
||||
BV: tl.constexpr,
|
||||
stride_qkv_token: tl.constexpr,
|
||||
stride_g_token: tl.constexpr,
|
||||
stride_beta_token,
|
||||
stride_out_token: tl.constexpr,
|
||||
stride_state_token: tl.constexpr,
|
||||
stride_indices_seq: tl.constexpr,
|
||||
IS_SPEC_DECODING: tl.constexpr,
|
||||
USE_QK_L2NORM_IN_KERNEL: tl.constexpr,
|
||||
USE_GATE_IN_KERNEL: tl.constexpr,
|
||||
APPLY_BETA_SIGMOID: tl.constexpr,
|
||||
HAS_DT_BIAS: tl.constexpr,
|
||||
USE_LOWER_BOUND: tl.constexpr,
|
||||
num_stages: tl.constexpr,
|
||||
launch_pdl: tl.constexpr,
|
||||
):
|
||||
if launch_pdl:
|
||||
tl.extra.cuda.gdc_wait()
|
||||
|
||||
pid = tl.program_id(0)
|
||||
i_v = pid % tl.cdiv(V, BV)
|
||||
i_nh = pid // tl.cdiv(V, BV)
|
||||
i_n, i_h = i_nh // H, i_nh % H
|
||||
bos = tl.load(cu_seqlens + i_n).to(tl.int64)
|
||||
eos = tl.load(cu_seqlens + i_n + 1).to(tl.int64)
|
||||
sequence_length = eos - bos
|
||||
if sequence_length == 0:
|
||||
return
|
||||
|
||||
o_k = tl.arange(0, BK)
|
||||
o_v = i_v * BV + tl.arange(0, BV)
|
||||
m_k = o_k < K
|
||||
m_v = o_v < V
|
||||
m_state = m_v[:, None] & m_k[None, :]
|
||||
|
||||
if IS_SPEC_DECODING:
|
||||
initial_token = tl.load(num_accepted_tokens + i_n).to(tl.int64) - 1
|
||||
else:
|
||||
initial_token = 0
|
||||
state_index = tl.load(state_indices + i_n * stride_indices_seq + initial_token).to(
|
||||
tl.int64
|
||||
)
|
||||
p_out = out + bos * stride_out_token + i_h * V + o_v
|
||||
if state_index <= 0:
|
||||
tl.store(p_out, tl.zeros([BV], dtype=tl.float32), mask=m_v)
|
||||
return
|
||||
|
||||
p_state = (
|
||||
state
|
||||
+ state_index * stride_state_token
|
||||
+ i_h * V * K
|
||||
+ o_v[:, None] * K
|
||||
+ o_k[None, :]
|
||||
)
|
||||
b_state = tl.load(p_state, mask=m_state, other=0.0).to(tl.float32)
|
||||
|
||||
p_q = q + bos * stride_qkv_token + i_h * K + o_k
|
||||
p_k = k + bos * stride_qkv_token + i_h * K + o_k
|
||||
p_v = v + bos * stride_qkv_token + i_h * V + o_v
|
||||
p_g = g + bos * stride_g_token + i_h * K + o_k
|
||||
p_beta = beta + bos * stride_beta_token + i_h
|
||||
for i_t in tl.range(0, sequence_length, num_stages=num_stages):
|
||||
b_q = tl.load(p_q, mask=m_k, other=0.0, eviction_policy="evict_last").to(
|
||||
tl.float32
|
||||
)
|
||||
b_k = tl.load(p_k, mask=m_k, other=0.0, eviction_policy="evict_last").to(
|
||||
tl.float32
|
||||
)
|
||||
b_v = tl.load(p_v, mask=m_v, other=0.0, eviction_policy="evict_first").to(
|
||||
tl.float32
|
||||
)
|
||||
if USE_QK_L2NORM_IN_KERNEL:
|
||||
b_q = b_q / tl.sqrt(tl.sum(b_q * b_q) + 1e-6)
|
||||
b_k = b_k / tl.sqrt(tl.sum(b_k * b_k) + 1e-6)
|
||||
b_q *= scale
|
||||
|
||||
b_gate = tl.load(
|
||||
p_g,
|
||||
mask=m_k,
|
||||
other=0.0,
|
||||
eviction_policy="evict_last",
|
||||
).to(tl.float32)
|
||||
if USE_GATE_IN_KERNEL:
|
||||
if HAS_DT_BIAS:
|
||||
b_bias = tl.load(
|
||||
dt_bias + i_h * K + o_k,
|
||||
mask=m_k,
|
||||
other=0.0,
|
||||
).to(tl.float32)
|
||||
b_gate += b_bias
|
||||
b_a = exp(tl.load(A_log + i_h).to(tl.float32))
|
||||
if USE_LOWER_BOUND:
|
||||
b_gate = lower_bound * tl.sigmoid(b_a * b_gate)
|
||||
else:
|
||||
b_softplus = tl.where(
|
||||
b_gate > 20.0,
|
||||
b_gate,
|
||||
log(1.0 + tl.exp(b_gate)),
|
||||
)
|
||||
b_gate = -b_a * b_softplus
|
||||
|
||||
b_state *= exp(b_gate[None, :])
|
||||
b_v -= tl.sum(b_state * b_k[None, :], axis=1)
|
||||
b_beta = tl.load(p_beta, eviction_policy="evict_last").to(tl.float32)
|
||||
if APPLY_BETA_SIGMOID:
|
||||
b_beta = tl.sigmoid(b_beta)
|
||||
b_v *= b_beta
|
||||
b_state += b_v[:, None] * b_k[None, :]
|
||||
b_out = tl.sum(b_state * b_q[None, :], axis=1)
|
||||
tl.store(
|
||||
p_out,
|
||||
b_out.to(p_out.dtype.element_ty),
|
||||
mask=m_v,
|
||||
eviction_policy="evict_first",
|
||||
)
|
||||
|
||||
final_state_index = tl.load(state_indices + i_n * stride_indices_seq + i_t).to(
|
||||
tl.int64
|
||||
)
|
||||
if final_state_index > 0:
|
||||
p_final_state = (
|
||||
state
|
||||
+ final_state_index * stride_state_token
|
||||
+ i_h * V * K
|
||||
+ o_v[:, None] * K
|
||||
+ o_k[None, :]
|
||||
)
|
||||
tl.store(
|
||||
p_final_state,
|
||||
b_state.to(p_final_state.dtype.element_ty),
|
||||
mask=m_state,
|
||||
)
|
||||
|
||||
p_q += stride_qkv_token
|
||||
p_k += stride_qkv_token
|
||||
p_v += stride_qkv_token
|
||||
p_g += stride_g_token
|
||||
p_beta += stride_beta_token
|
||||
p_out += stride_out_token
|
||||
|
||||
if launch_pdl:
|
||||
tl.extra.cuda.gdc_launch_dependents()
|
||||
|
||||
|
||||
# Consumed by kimi_k3_triton_warmup.py during kernel_warmup().
|
||||
def get_fused_recurrent_kda_fwd_warmup_profiles(
|
||||
num_heads: int,
|
||||
) -> tuple[int, ...]:
|
||||
"""Return representative sequence counts for gated launch variants."""
|
||||
# The region above 192 head-sequences reuses the second launch variant.
|
||||
return (
|
||||
1,
|
||||
48 // num_heads + 1,
|
||||
96 // num_heads + 1,
|
||||
)
|
||||
|
||||
|
||||
def fused_recurrent_kda_fwd(
|
||||
q: torch.Tensor,
|
||||
k: torch.Tensor,
|
||||
v: torch.Tensor,
|
||||
g: torch.Tensor,
|
||||
beta: torch.Tensor,
|
||||
scale: float | None = None,
|
||||
initial_state: torch.Tensor | None = None,
|
||||
inplace_final_state: bool = True,
|
||||
cu_seqlens: torch.Tensor | None = None,
|
||||
ssm_state_indices: torch.Tensor | None = None,
|
||||
num_accepted_tokens: torch.Tensor | None = None,
|
||||
use_qk_l2norm_in_kernel: bool = True,
|
||||
A_log: torch.Tensor | None = None,
|
||||
dt_bias: torch.Tensor | None = None,
|
||||
lower_bound: float | None = None,
|
||||
use_gate_in_kernel: bool = False,
|
||||
use_beta_sigmoid_in_kernel: bool = False,
|
||||
out: torch.Tensor | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Launch recurrent KDA with dense inner dimensions and row strides."""
|
||||
B, T, H, K = q.shape
|
||||
V = v.shape[-1]
|
||||
assert B == 1 and k.shape == q.shape
|
||||
assert v.shape == (B, T, H, V) and g.shape == (B, T, H, K)
|
||||
assert beta.shape == (B, T, H)
|
||||
assert initial_state is not None
|
||||
assert cu_seqlens is not None
|
||||
assert ssm_state_indices is not None
|
||||
assert inplace_final_state
|
||||
if out is None:
|
||||
out = torch.empty_like(v)
|
||||
assert out.shape == v.shape
|
||||
assert initial_state.shape[1:] == (H, V, K)
|
||||
assert ssm_state_indices.ndim in (1, 2)
|
||||
|
||||
assert q.stride()[2:] == k.stride()[2:] == (K, 1)
|
||||
assert v.stride()[2:] == out.stride()[2:] == (V, 1)
|
||||
assert g.stride()[2:] == (K, 1)
|
||||
assert beta.stride(2) == 1
|
||||
assert q.stride(1) == k.stride(1) == v.stride(1)
|
||||
assert initial_state.stride()[1:] == (V * K, K, 1)
|
||||
N = cu_seqlens.numel() - 1
|
||||
if ssm_state_indices.ndim == 1:
|
||||
assert T == N
|
||||
assert num_accepted_tokens is None
|
||||
else:
|
||||
assert ssm_state_indices.stride(1) == 1
|
||||
assert cu_seqlens.is_contiguous()
|
||||
if use_gate_in_kernel:
|
||||
assert A_log is not None and A_log.is_contiguous()
|
||||
assert dt_bias is None or dt_bias.is_contiguous()
|
||||
|
||||
if scale is None:
|
||||
scale = K**-0.5
|
||||
|
||||
if use_gate_in_kernel:
|
||||
# Tuned on GB300 for Kimi-K3 shapes. Keep the warmup profiles above in
|
||||
# sync with these boundaries.
|
||||
head_sequences = H * N
|
||||
if head_sequences <= 48:
|
||||
BV, num_stages = 4, 4
|
||||
elif head_sequences <= 96:
|
||||
BV, num_stages = 8, 3
|
||||
elif head_sequences <= 192:
|
||||
BV, num_stages = 16, 3
|
||||
else:
|
||||
BV, num_stages = 8, 3
|
||||
num_warps = 1
|
||||
else:
|
||||
BV, num_warps, num_stages = 8, 1, 2
|
||||
grid = (cdiv(V, BV) * N * H,)
|
||||
fused_recurrent_kda_fwd_kernel[grid](
|
||||
q=q,
|
||||
k=k,
|
||||
v=v,
|
||||
g=g,
|
||||
beta=beta,
|
||||
A_log=A_log,
|
||||
dt_bias=dt_bias,
|
||||
out=out,
|
||||
state=initial_state,
|
||||
cu_seqlens=cu_seqlens,
|
||||
state_indices=ssm_state_indices,
|
||||
num_accepted_tokens=num_accepted_tokens,
|
||||
lower_bound=lower_bound,
|
||||
scale=scale,
|
||||
N=N,
|
||||
T=T,
|
||||
H=H,
|
||||
K=K,
|
||||
V=V,
|
||||
BK=next_power_of_2(K),
|
||||
BV=BV,
|
||||
stride_qkv_token=q.stride(1),
|
||||
stride_g_token=g.stride(1),
|
||||
stride_beta_token=beta.stride(1),
|
||||
stride_out_token=out.stride(1),
|
||||
stride_state_token=initial_state.stride(0),
|
||||
stride_indices_seq=ssm_state_indices.stride(0),
|
||||
IS_SPEC_DECODING=num_accepted_tokens is not None,
|
||||
USE_QK_L2NORM_IN_KERNEL=use_qk_l2norm_in_kernel,
|
||||
USE_GATE_IN_KERNEL=use_gate_in_kernel,
|
||||
APPLY_BETA_SIGMOID=use_beta_sigmoid_in_kernel,
|
||||
num_warps=num_warps,
|
||||
num_stages=num_stages,
|
||||
launch_pdl=current_platform.is_arch_support_pdl(),
|
||||
)
|
||||
return out, initial_state
|
||||
|
||||
|
||||
def fused_recurrent_kda(
|
||||
q: torch.Tensor,
|
||||
k: torch.Tensor,
|
||||
v: torch.Tensor,
|
||||
raw_g: torch.Tensor,
|
||||
raw_beta: torch.Tensor,
|
||||
A_log: torch.Tensor,
|
||||
dt_bias: torch.Tensor | None,
|
||||
lower_bound: float | None,
|
||||
initial_state: torch.Tensor,
|
||||
cu_seqlens: torch.Tensor,
|
||||
ssm_state_indices: torch.Tensor,
|
||||
num_accepted_tokens: torch.Tensor | None = None,
|
||||
out: torch.Tensor | None = None,
|
||||
fuse_gate: bool | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Run recurrent KDA from raw gate and beta inputs.
|
||||
|
||||
This vLLM wrapper applies the gate activation and beta sigmoid, selecting
|
||||
whether to materialize them before launching the recurrent kernel.
|
||||
"""
|
||||
if fuse_gate is None:
|
||||
fuse_gate = True
|
||||
|
||||
if fuse_gate:
|
||||
gate = raw_g
|
||||
beta = raw_beta
|
||||
else:
|
||||
gate, beta = _fused_kda_gate_beta(
|
||||
raw_g,
|
||||
raw_beta,
|
||||
A_log,
|
||||
dt_bias,
|
||||
lower_bound,
|
||||
)
|
||||
return fused_recurrent_kda_fwd(
|
||||
q=q,
|
||||
k=k,
|
||||
v=v,
|
||||
g=gate,
|
||||
beta=beta,
|
||||
scale=q.shape[-1] ** -0.5,
|
||||
initial_state=initial_state,
|
||||
inplace_final_state=True,
|
||||
cu_seqlens=cu_seqlens,
|
||||
ssm_state_indices=ssm_state_indices,
|
||||
num_accepted_tokens=num_accepted_tokens,
|
||||
use_qk_l2norm_in_kernel=True,
|
||||
A_log=A_log if fuse_gate else None,
|
||||
dt_bias=dt_bias if fuse_gate else None,
|
||||
lower_bound=lower_bound if fuse_gate else None,
|
||||
use_gate_in_kernel=fuse_gate,
|
||||
use_beta_sigmoid_in_kernel=fuse_gate,
|
||||
out=out,
|
||||
)
|
||||
|
||||
|
||||
@triton.jit(
|
||||
do_not_specialize=["stride_beta_token", "stride_state_indices"]
|
||||
)
|
||||
def fused_recurrent_kda_packed_decode_kernel(
|
||||
mixed_qkv,
|
||||
raw_g,
|
||||
raw_beta,
|
||||
A_log,
|
||||
dt_bias,
|
||||
out,
|
||||
state,
|
||||
state_indices,
|
||||
lower_bound,
|
||||
scale: tl.constexpr,
|
||||
stride_mixed_token: tl.constexpr,
|
||||
stride_g_token: tl.constexpr,
|
||||
stride_beta_token,
|
||||
stride_state_token: tl.constexpr,
|
||||
stride_state_indices,
|
||||
H: tl.constexpr,
|
||||
K: tl.constexpr,
|
||||
V: tl.constexpr,
|
||||
BK: tl.constexpr,
|
||||
BV: tl.constexpr,
|
||||
SOFTPLUS_THRESHOLD: tl.constexpr,
|
||||
USE_LOWER_BOUND: tl.constexpr,
|
||||
launch_pdl: tl.constexpr,
|
||||
):
|
||||
i_v, i_nh = tl.program_id(0), tl.program_id(1)
|
||||
i_n, i_h = i_nh // H, i_nh % H
|
||||
|
||||
o_k = tl.arange(0, BK)
|
||||
o_v = i_v * BV + tl.arange(0, BV)
|
||||
mask_k = o_k < K
|
||||
mask_v = o_v < V
|
||||
mask_state = mask_v[:, None] & mask_k[None, :]
|
||||
|
||||
if launch_pdl:
|
||||
tl.extra.cuda.gdc_wait()
|
||||
tl.extra.cuda.gdc_launch_dependents()
|
||||
|
||||
state_idx = tl.load(state_indices + i_n * stride_state_indices).to(tl.int64)
|
||||
p_out = out + (i_n * H + i_h) * V + o_v
|
||||
if state_idx <= 0:
|
||||
tl.store(p_out, tl.zeros([BV], dtype=tl.float32), mask=mask_v)
|
||||
return
|
||||
|
||||
p_state = state + state_idx * stride_state_token
|
||||
p_state += i_h * V * K + o_v[:, None] * K + o_k[None, :]
|
||||
b_state = tl.load(p_state, mask=mask_state, other=0).to(tl.float32)
|
||||
|
||||
# Q, K, and V occupy consecutive channel ranges, while the token stride
|
||||
# may also include the output-gate projection that follows packed QKV.
|
||||
p_mixed = mixed_qkv + i_n * stride_mixed_token
|
||||
b_q = tl.load(p_mixed + i_h * K + o_k, mask=mask_k, other=0).to(tl.float32)
|
||||
b_k = tl.load(
|
||||
p_mixed + H * K + i_h * K + o_k,
|
||||
mask=mask_k,
|
||||
other=0,
|
||||
).to(tl.float32)
|
||||
b_v = tl.load(
|
||||
p_mixed + 2 * H * K + i_h * V + o_v,
|
||||
mask=mask_v,
|
||||
other=0,
|
||||
).to(tl.float32)
|
||||
|
||||
b_q /= tl.sqrt(tl.sum(b_q * b_q) + 1e-6)
|
||||
b_k /= tl.sqrt(tl.sum(b_k * b_k) + 1e-6)
|
||||
b_q *= scale
|
||||
|
||||
p_g = raw_g + i_n * stride_g_token + i_h * K + o_k
|
||||
b_g = tl.load(p_g, mask=mask_k, other=0).to(tl.float32)
|
||||
b_bias = tl.load(dt_bias + i_h * K + o_k, mask=mask_k, other=0).to(tl.float32)
|
||||
b_a = exp(tl.load(A_log + i_h).to(tl.float32))
|
||||
b_g += b_bias
|
||||
if USE_LOWER_BOUND:
|
||||
b_gate = lower_bound * tl.sigmoid(b_a * b_g)
|
||||
else:
|
||||
b_softplus = tl.where(
|
||||
b_g > SOFTPLUS_THRESHOLD,
|
||||
b_g,
|
||||
log(1.0 + tl.exp(b_g)),
|
||||
)
|
||||
b_gate = -b_a * b_softplus
|
||||
|
||||
b_state *= exp(b_gate[None, :])
|
||||
b_v -= tl.sum(b_state * b_k[None, :], axis=1)
|
||||
b_beta = tl.sigmoid(
|
||||
tl.load(raw_beta + i_n * stride_beta_token + i_h).to(tl.float32)
|
||||
)
|
||||
b_v *= b_beta
|
||||
b_state += b_v[:, None] * b_k[None, :]
|
||||
b_out = tl.sum(b_state * b_q[None, :], axis=1)
|
||||
|
||||
tl.store(p_out, b_out.to(p_out.dtype.element_ty), mask=mask_v)
|
||||
tl.store(p_state, b_state.to(p_state.dtype.element_ty), mask=mask_state)
|
||||
|
||||
|
||||
def fused_recurrent_kda_packed_decode(
|
||||
mixed_qkv: torch.Tensor,
|
||||
raw_g: torch.Tensor,
|
||||
raw_beta: torch.Tensor,
|
||||
A_log: torch.Tensor,
|
||||
dt_bias: torch.Tensor,
|
||||
lower_bound: float | None,
|
||||
initial_state: torch.Tensor,
|
||||
state_indices: torch.Tensor,
|
||||
scale: float | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Run one-token KDA decode directly from packed post-conv QKV."""
|
||||
if mixed_qkv.ndim != 2 or mixed_qkv.stride(-1) != 1:
|
||||
raise ValueError("`mixed_qkv` must be 2D and contiguous in its last dim.")
|
||||
if raw_g.ndim != 4 or raw_g.shape[0] != 1:
|
||||
raise ValueError("`raw_g` must have shape [1, B, H, K].")
|
||||
if raw_beta.ndim != 3 or raw_beta.shape[0] != 1:
|
||||
raise ValueError("`raw_beta` must have shape [1, B, H].")
|
||||
if initial_state.ndim != 4:
|
||||
raise ValueError("`initial_state` must have shape [cache, H, V, K].")
|
||||
_, H, V, K = initial_state.shape
|
||||
if raw_g.stride()[2:] != (K, 1):
|
||||
raise ValueError("`raw_g` must be contiguous within each token.")
|
||||
if raw_beta.stride(2) != 1:
|
||||
raise ValueError("`raw_beta` heads must be contiguous.")
|
||||
if initial_state.stride()[1:] != (V * K, K, 1):
|
||||
raise ValueError("`initial_state` must be contiguous within each cache slot.")
|
||||
if state_indices.ndim != 1:
|
||||
raise ValueError("`state_indices` must be one-dimensional.")
|
||||
if A_log.ndim != 1 or not A_log.is_contiguous():
|
||||
raise ValueError("`A_log` must be contiguous and one-dimensional.")
|
||||
if not dt_bias.is_contiguous():
|
||||
raise ValueError("`dt_bias` must be contiguous.")
|
||||
|
||||
device = mixed_qkv.device
|
||||
if any(
|
||||
x.device != device
|
||||
for x in (raw_g, raw_beta, A_log, dt_bias, initial_state, state_indices)
|
||||
):
|
||||
raise ValueError("All packed KDA inputs must be on the same device.")
|
||||
|
||||
B = mixed_qkv.shape[0]
|
||||
if raw_g.shape != (1, B, H, K):
|
||||
raise ValueError(f"Unexpected raw gate shape {tuple(raw_g.shape)}.")
|
||||
if raw_beta.shape != (1, B, H):
|
||||
raise ValueError(f"Unexpected raw beta shape {tuple(raw_beta.shape)}.")
|
||||
if mixed_qkv.shape[1] != 2 * H * K + H * V:
|
||||
raise ValueError(f"Unexpected packed QKV shape {tuple(mixed_qkv.shape)}.")
|
||||
if A_log.numel() != H or dt_bias.numel() != H * K:
|
||||
raise ValueError("`A_log` or `dt_bias` has an incompatible shape.")
|
||||
if state_indices.shape[0] != B:
|
||||
raise ValueError("`state_indices` must contain one entry per token.")
|
||||
|
||||
BK = next_power_of_2(K)
|
||||
BV = min(next_power_of_2(V), 32)
|
||||
if scale is None:
|
||||
scale = K**-0.5
|
||||
|
||||
out = torch.empty((1, B, H, V), dtype=mixed_qkv.dtype, device=device)
|
||||
grid = (cdiv(V, BV), B * H)
|
||||
fused_recurrent_kda_packed_decode_kernel[grid](
|
||||
mixed_qkv=mixed_qkv,
|
||||
raw_g=raw_g,
|
||||
raw_beta=raw_beta,
|
||||
A_log=A_log,
|
||||
dt_bias=dt_bias,
|
||||
out=out,
|
||||
state=initial_state,
|
||||
state_indices=state_indices,
|
||||
lower_bound=lower_bound or 0.0,
|
||||
scale=scale,
|
||||
stride_mixed_token=mixed_qkv.stride(0),
|
||||
stride_g_token=raw_g.stride(1),
|
||||
stride_beta_token=raw_beta.stride(1),
|
||||
stride_state_token=initial_state.stride(0),
|
||||
stride_state_indices=state_indices.stride(0),
|
||||
H=H,
|
||||
K=K,
|
||||
V=V,
|
||||
BK=BK,
|
||||
BV=BV,
|
||||
SOFTPLUS_THRESHOLD=20.0,
|
||||
USE_LOWER_BOUND=lower_bound is not None,
|
||||
num_warps=4,
|
||||
num_stages=2,
|
||||
launch_pdl=current_platform.is_arch_support_pdl(),
|
||||
)
|
||||
return out, initial_state
|
||||
@@ -0,0 +1,195 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Startup compilation of Kimi-K3 vision FA4 kernels."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
from collections.abc import Iterator
|
||||
from dataclasses import dataclass
|
||||
from functools import partial
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.warmup.cutedsl_warmup import (
|
||||
CuTeDSLCompileUnit,
|
||||
register_cutedsl_warmup_provider,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
_FA4_TILE_SIZE = 128
|
||||
_FA4_MAX_SPLITS = 128
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class KimiK3VisionFA4WarmupConfig:
|
||||
num_heads: int
|
||||
head_dim: int
|
||||
dtype: torch.dtype
|
||||
max_batch_size: int
|
||||
max_seqlen: int
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class _FA4WarmupProbe:
|
||||
batch_size: int
|
||||
max_seqlen: int
|
||||
dispatch_key: tuple[int, bool, int | None]
|
||||
|
||||
|
||||
def _combine_log_max_splits(head_dim: int, num_splits: int) -> int:
|
||||
k_block_size = 64 if head_dim <= 64 else 128
|
||||
tile_m = 8 if k_block_size == 128 else 16
|
||||
min_log_max_splits = 5 if tile_m == 8 else 4
|
||||
return max(math.ceil(math.log2(num_splits)), min_log_max_splits)
|
||||
|
||||
|
||||
def _get_dispatch(
|
||||
config: KimiK3VisionFA4WarmupConfig,
|
||||
*,
|
||||
batch_size: int,
|
||||
max_seqlen: int,
|
||||
num_sms: int,
|
||||
) -> tuple[int, bool, int | None]:
|
||||
# These are the shape-dependent fields in FA4's forward and combine
|
||||
# compile keys for noncausal varlen MHA. Compile units still pass
|
||||
# num_splits=0 so the production selector makes the final decision.
|
||||
q_stage = 2 if max_seqlen > _FA4_TILE_SIZE else 1
|
||||
effective_q_tile = q_stage * _FA4_TILE_SIZE
|
||||
num_m_blocks = math.ceil(max_seqlen / effective_q_tile)
|
||||
num_n_blocks = math.ceil(max_seqlen / _FA4_TILE_SIZE)
|
||||
|
||||
if num_n_blocks <= 4:
|
||||
num_splits = 1
|
||||
else:
|
||||
total_m_blocks = batch_size * config.num_heads * num_m_blocks
|
||||
num_splits = min(
|
||||
num_sms // total_m_blocks,
|
||||
_FA4_MAX_SPLITS,
|
||||
num_n_blocks,
|
||||
)
|
||||
|
||||
is_split_kv = num_splits > 1
|
||||
combine_key = (
|
||||
_combine_log_max_splits(config.head_dim, num_splits) if is_split_kv else None
|
||||
)
|
||||
return q_stage, is_split_kv, combine_key
|
||||
|
||||
|
||||
def _get_warmup_probes(
|
||||
config: KimiK3VisionFA4WarmupConfig,
|
||||
*,
|
||||
num_sms: int,
|
||||
) -> tuple[_FA4WarmupProbe, ...]:
|
||||
if config.max_batch_size <= 0 or config.max_seqlen <= 0:
|
||||
return ()
|
||||
|
||||
probes: dict[tuple[int, bool, int | None], _FA4WarmupProbe] = {}
|
||||
|
||||
def add_probe(batch_size: int, max_seqlen: int) -> None:
|
||||
dispatch_key = _get_dispatch(
|
||||
config,
|
||||
batch_size=batch_size,
|
||||
max_seqlen=max_seqlen,
|
||||
num_sms=num_sms,
|
||||
)
|
||||
if dispatch_key not in probes:
|
||||
probes[dispatch_key] = _FA4WarmupProbe(
|
||||
batch_size=batch_size,
|
||||
max_seqlen=max_seqlen,
|
||||
dispatch_key=dispatch_key,
|
||||
)
|
||||
|
||||
add_probe(batch_size=1, max_seqlen=1)
|
||||
if config.max_seqlen > _FA4_TILE_SIZE:
|
||||
add_probe(batch_size=1, max_seqlen=_FA4_TILE_SIZE + 1)
|
||||
|
||||
max_n_blocks = math.ceil(config.max_seqlen / _FA4_TILE_SIZE)
|
||||
for num_n_blocks in range(5, max_n_blocks + 1):
|
||||
max_seqlen = min(
|
||||
(num_n_blocks - 1) * _FA4_TILE_SIZE + 1,
|
||||
config.max_seqlen,
|
||||
)
|
||||
num_m_blocks = math.ceil(max_seqlen / (2 * _FA4_TILE_SIZE))
|
||||
max_split_batch_size = min(
|
||||
config.max_batch_size,
|
||||
num_sms // (2 * config.num_heads * num_m_blocks),
|
||||
)
|
||||
if max_split_batch_size == 0:
|
||||
break
|
||||
for batch_size in range(1, max_split_batch_size + 1):
|
||||
add_probe(batch_size, max_seqlen)
|
||||
|
||||
return tuple(probes.values())
|
||||
|
||||
|
||||
def _iter_compile_units(
|
||||
config: KimiK3VisionFA4WarmupConfig,
|
||||
) -> Iterator[CuTeDSLCompileUnit]:
|
||||
device_id = current_platform.current_device()
|
||||
num_sms = current_platform.num_compute_units(device_id)
|
||||
for probe in _get_warmup_probes(config, num_sms=num_sms):
|
||||
yield CuTeDSLCompileUnit(
|
||||
name="kimi_k3_vision_fa4",
|
||||
key=("kimi_k3_vision_fa4", config, probe.dispatch_key),
|
||||
compile=partial(_compile, config, probe),
|
||||
)
|
||||
|
||||
|
||||
def _compile(
|
||||
config: KimiK3VisionFA4WarmupConfig,
|
||||
probe: _FA4WarmupProbe,
|
||||
) -> None:
|
||||
from vllm.vllm_flash_attn import flash_attn_varlen_func
|
||||
|
||||
device = current_platform.current_device()
|
||||
total_tokens = probe.batch_size * probe.max_seqlen
|
||||
qkv = torch.empty(
|
||||
3,
|
||||
total_tokens,
|
||||
config.num_heads,
|
||||
config.head_dim,
|
||||
device=device,
|
||||
dtype=config.dtype,
|
||||
)
|
||||
cu_seqlens = torch.arange(
|
||||
0,
|
||||
total_tokens + 1,
|
||||
probe.max_seqlen,
|
||||
device=qkv.device,
|
||||
dtype=torch.int32,
|
||||
)
|
||||
flash_attn_varlen_func(
|
||||
qkv[0],
|
||||
qkv[1],
|
||||
qkv[2],
|
||||
cu_seqlens_q=cu_seqlens,
|
||||
cu_seqlens_k=cu_seqlens,
|
||||
max_seqlen_q=probe.max_seqlen,
|
||||
max_seqlen_k=probe.max_seqlen,
|
||||
dropout_p=0.0,
|
||||
causal=False,
|
||||
softmax_scale=config.head_dim**-0.5,
|
||||
fa_version=4,
|
||||
num_splits=0,
|
||||
)
|
||||
|
||||
|
||||
class _WarmupProvider:
|
||||
def __init__(self) -> None:
|
||||
self.configs: set[KimiK3VisionFA4WarmupConfig] = set()
|
||||
|
||||
def get_cutedsl_warmup_compile_units(self) -> tuple[CuTeDSLCompileUnit, ...]:
|
||||
return tuple(
|
||||
unit for config in self.configs for unit in _iter_compile_units(config)
|
||||
)
|
||||
|
||||
|
||||
_PROVIDER = _WarmupProvider()
|
||||
|
||||
|
||||
def register_kimi_k3_vision_fa4_warmup(
|
||||
config: KimiK3VisionFA4WarmupConfig,
|
||||
) -> None:
|
||||
_PROVIDER.configs.add(config)
|
||||
register_cutedsl_warmup_provider(_PROVIDER)
|
||||
@@ -0,0 +1,412 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# SPDX-FileCopyrightText: Songlin Yang, Yu Zhang
|
||||
#
|
||||
# This file contains code copied from the flash-linear-attention project.
|
||||
# The original source was licensed under the MIT license.
|
||||
# Copyright (c) 2023-2025, Songlin Yang, Yu Zhang
|
||||
# ruff: noqa: E501
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
from vllm.model_executor.custom_op import CustomOp
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.utils.math_utils import cdiv, next_power_of_2
|
||||
|
||||
@triton.heuristics(
|
||||
{
|
||||
"STORE_RESIDUAL_OUT": lambda args: args["residual_out"] is not None,
|
||||
"HAS_RESIDUAL": lambda args: args["residual"] is not None,
|
||||
"HAS_WEIGHT": lambda args: args["w"] is not None,
|
||||
"HAS_BIAS": lambda args: args["b"] is not None,
|
||||
}
|
||||
)
|
||||
@triton.jit
|
||||
def layer_norm_gated_fwd_kernel(
|
||||
x, # pointer to the input
|
||||
g, # pointer to the gate
|
||||
y, # pointer to the output
|
||||
w, # pointer to the weights
|
||||
b, # pointer to the biases
|
||||
residual, # pointer to the residual
|
||||
residual_out, # pointer to the residual
|
||||
mean, # pointer to the mean
|
||||
rstd, # pointer to the 1/std
|
||||
eps, # epsilon to avoid division by zero
|
||||
T, # number of rows in x
|
||||
H: tl.constexpr, # number of heads
|
||||
g_stride_n,
|
||||
D: tl.constexpr, # number of columns in x
|
||||
BT: tl.constexpr,
|
||||
BD: tl.constexpr,
|
||||
ACTIVATION: tl.constexpr,
|
||||
IS_RMS_NORM: tl.constexpr,
|
||||
STORE_RESIDUAL_OUT: tl.constexpr,
|
||||
HAS_RESIDUAL: tl.constexpr,
|
||||
HAS_WEIGHT: tl.constexpr,
|
||||
HAS_BIAS: tl.constexpr,
|
||||
launch_pdl: tl.constexpr,
|
||||
):
|
||||
i_t = tl.program_id(0)
|
||||
|
||||
o_d = tl.arange(0, BD)
|
||||
m_d = o_d < D
|
||||
|
||||
if launch_pdl:
|
||||
tl.extra.cuda.gdc_wait()
|
||||
tl.extra.cuda.gdc_launch_dependents()
|
||||
|
||||
p_x = tl.make_block_ptr(x, (T, D), (D, 1), (i_t * BT, 0), (BT, BD), (1, 0))
|
||||
b_x = tl.load(p_x, boundary_check=(0, 1)).to(tl.float32)
|
||||
if HAS_RESIDUAL:
|
||||
p_res = tl.make_block_ptr(
|
||||
residual, (T, D), (D, 1), (i_t * BT, 0), (BT, BD), (1, 0)
|
||||
)
|
||||
b_x += tl.load(p_res, boundary_check=(0, 1)).to(tl.float32)
|
||||
if STORE_RESIDUAL_OUT:
|
||||
p_res_out = tl.make_block_ptr(
|
||||
residual_out, (T, D), (D, 1), (i_t * BT, 0), (BT, BD), (1, 0)
|
||||
)
|
||||
tl.store(p_res_out, b_x.to(p_res_out.dtype.element_ty), boundary_check=(0, 1))
|
||||
if not IS_RMS_NORM:
|
||||
b_mean = tl.sum(b_x, axis=1) / D
|
||||
p_mean = tl.make_block_ptr(mean, (T,), (1,), (i_t * BT,), (BT,), (0,))
|
||||
tl.store(p_mean, b_mean.to(p_mean.dtype.element_ty), boundary_check=(0,))
|
||||
b_xbar = tl.where(m_d[None, :], b_x - b_mean[:, None], 0.0)
|
||||
b_var = tl.sum(b_xbar * b_xbar, axis=1) / D
|
||||
else:
|
||||
b_xbar = tl.where(m_d[None, :], b_x, 0.0)
|
||||
b_var = tl.sum(b_xbar * b_xbar, axis=1) / D
|
||||
b_rstd = 1 / tl.sqrt(b_var + eps)
|
||||
|
||||
p_rstd = tl.make_block_ptr(rstd, (T,), (1,), (i_t * BT,), (BT,), (0,))
|
||||
tl.store(p_rstd, b_rstd.to(p_rstd.dtype.element_ty), boundary_check=(0,))
|
||||
|
||||
if HAS_WEIGHT:
|
||||
b_w = tl.load(w + o_d, mask=m_d).to(tl.float32)
|
||||
if HAS_BIAS:
|
||||
b_b = tl.load(b + o_d, mask=m_d).to(tl.float32)
|
||||
b_x_hat = (
|
||||
(b_x - b_mean[:, None]) * b_rstd[:, None]
|
||||
if not IS_RMS_NORM
|
||||
else b_x * b_rstd[:, None]
|
||||
)
|
||||
b_y = b_x_hat * b_w[None, :] if HAS_WEIGHT else b_x_hat
|
||||
if HAS_BIAS:
|
||||
b_y = b_y + b_b[None, :]
|
||||
|
||||
# swish/sigmoid output gate
|
||||
o_t = i_t * BT + tl.arange(0, BT)
|
||||
o_g = (o_t // H) * g_stride_n + (o_t % H) * D
|
||||
b_g = tl.load(
|
||||
g + o_g[:, None] + o_d[None, :],
|
||||
mask=(o_t[:, None] < T) & m_d[None, :],
|
||||
other=0.0,
|
||||
).to(tl.float32)
|
||||
if ACTIVATION == "swish" or ACTIVATION == "silu":
|
||||
b_y = b_y * b_g * tl.sigmoid(b_g)
|
||||
elif ACTIVATION == "sigmoid":
|
||||
b_y = b_y * tl.sigmoid(b_g)
|
||||
|
||||
# Write output
|
||||
p_y = tl.make_block_ptr(y, (T, D), (D, 1), (i_t * BT, 0), (BT, BD), (1, 0))
|
||||
tl.store(p_y, b_y.to(p_y.dtype.element_ty), boundary_check=(0, 1))
|
||||
|
||||
|
||||
@triton.heuristics(
|
||||
{
|
||||
"STORE_RESIDUAL_OUT": lambda args: args["residual_out"] is not None,
|
||||
"HAS_RESIDUAL": lambda args: args["residual"] is not None,
|
||||
"HAS_WEIGHT": lambda args: args["w"] is not None,
|
||||
"HAS_BIAS": lambda args: args["b"] is not None,
|
||||
}
|
||||
)
|
||||
@triton.jit
|
||||
def layer_norm_gated_fwd_kernel1(
|
||||
x, # pointer to the input
|
||||
g, # pointer to the gate
|
||||
y, # pointer to the output
|
||||
w, # pointer to the weights
|
||||
b, # pointer to the biases
|
||||
residual, # pointer to the residual
|
||||
residual_out, # pointer to the residual
|
||||
mean, # pointer to the mean
|
||||
rstd, # pointer to the 1/std
|
||||
eps, # epsilon to avoid division by zero
|
||||
D: tl.constexpr, # number of columns in x
|
||||
BD: tl.constexpr,
|
||||
ACTIVATION: tl.constexpr,
|
||||
IS_RMS_NORM: tl.constexpr,
|
||||
STORE_RESIDUAL_OUT: tl.constexpr,
|
||||
HAS_RESIDUAL: tl.constexpr,
|
||||
HAS_WEIGHT: tl.constexpr,
|
||||
HAS_BIAS: tl.constexpr,
|
||||
launch_pdl: tl.constexpr,
|
||||
):
|
||||
i_t = tl.program_id(0)
|
||||
x += i_t * D
|
||||
y += i_t * D
|
||||
g += i_t * D
|
||||
if HAS_RESIDUAL:
|
||||
residual += i_t * D
|
||||
if STORE_RESIDUAL_OUT:
|
||||
residual_out += i_t * D
|
||||
|
||||
if launch_pdl:
|
||||
tl.extra.cuda.gdc_wait()
|
||||
tl.extra.cuda.gdc_launch_dependents()
|
||||
|
||||
o_d = tl.arange(0, BD)
|
||||
m_d = o_d < D
|
||||
b_x = tl.load(x + o_d, mask=m_d, other=0.0).to(tl.float32)
|
||||
if HAS_RESIDUAL:
|
||||
b_x += tl.load(residual + o_d, mask=m_d, other=0.0).to(tl.float32)
|
||||
if STORE_RESIDUAL_OUT:
|
||||
tl.store(residual_out + o_d, b_x, mask=m_d)
|
||||
if not IS_RMS_NORM:
|
||||
b_mean = tl.sum(b_x, axis=0) / D
|
||||
tl.store(mean + i_t, b_mean)
|
||||
b_xbar = tl.where(m_d, b_x - b_mean, 0.0)
|
||||
b_var = tl.sum(b_xbar * b_xbar, axis=0) / D
|
||||
else:
|
||||
b_xbar = tl.where(m_d, b_x, 0.0)
|
||||
b_var = tl.sum(b_xbar * b_xbar, axis=0) / D
|
||||
b_rstd = 1 / tl.sqrt(b_var + eps)
|
||||
tl.store(rstd + i_t, b_rstd)
|
||||
|
||||
if HAS_WEIGHT:
|
||||
b_w = tl.load(w + o_d, mask=m_d).to(tl.float32)
|
||||
if HAS_BIAS:
|
||||
b_b = tl.load(b + o_d, mask=m_d).to(tl.float32)
|
||||
b_x_hat = (b_x - b_mean) * b_rstd if not IS_RMS_NORM else b_x * b_rstd
|
||||
b_y = b_x_hat * b_w if HAS_WEIGHT else b_x_hat
|
||||
if HAS_BIAS:
|
||||
b_y = b_y + b_b
|
||||
|
||||
# swish/sigmoid output gate
|
||||
b_g = tl.load(g + o_d, mask=m_d, other=0.0).to(tl.float32)
|
||||
if ACTIVATION == "swish" or ACTIVATION == "silu":
|
||||
b_y = b_y * b_g * tl.sigmoid(b_g)
|
||||
elif ACTIVATION == "sigmoid":
|
||||
b_y = b_y * tl.sigmoid(b_g)
|
||||
|
||||
# Write output
|
||||
tl.store(y + o_d, b_y, mask=m_d)
|
||||
|
||||
|
||||
def layer_norm_gated_fwd(
|
||||
x: torch.Tensor,
|
||||
g: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
bias: torch.Tensor,
|
||||
activation: str = "swish",
|
||||
eps: float = 1e-5,
|
||||
residual: torch.Tensor = None,
|
||||
out_dtype: torch.dtype = None,
|
||||
residual_dtype: torch.dtype = None,
|
||||
is_rms_norm: bool = False,
|
||||
H: int = 1,
|
||||
g_stride_n: int | None = None,
|
||||
):
|
||||
if residual is not None:
|
||||
residual_dtype = residual.dtype
|
||||
T, D = x.shape
|
||||
if g_stride_n is None:
|
||||
g_stride_n = D
|
||||
assert T % H == 0
|
||||
if residual is not None:
|
||||
assert residual.shape == (T, D)
|
||||
if weight is not None:
|
||||
assert weight.shape == (D,)
|
||||
if bias is not None:
|
||||
assert bias.shape == (D,)
|
||||
# allocate output
|
||||
y = x if out_dtype is None else torch.empty_like(x, dtype=out_dtype)
|
||||
if residual is not None or (
|
||||
residual_dtype is not None and residual_dtype != x.dtype
|
||||
):
|
||||
residual_out = torch.empty(T, D, device=x.device, dtype=residual_dtype)
|
||||
else:
|
||||
residual_out = None
|
||||
mean = (
|
||||
torch.empty((T,), dtype=torch.float, device=x.device)
|
||||
if not is_rms_norm
|
||||
else None
|
||||
)
|
||||
rstd = torch.empty((T,), dtype=torch.float, device=x.device)
|
||||
# Less than 64KB per feature: enqueue fused kernel
|
||||
MAX_FUSED_SIZE = 65536 // x.element_size()
|
||||
BD = min(MAX_FUSED_SIZE, next_power_of_2(D))
|
||||
if D > BD:
|
||||
raise RuntimeError("This layer norm doesn't support feature dim >= 64KB.")
|
||||
if D <= 512:
|
||||
BT = 16
|
||||
layer_norm_gated_fwd_kernel[(cdiv(T, BT),)](
|
||||
x=x,
|
||||
g=g,
|
||||
y=y,
|
||||
w=weight,
|
||||
b=bias,
|
||||
residual=residual,
|
||||
residual_out=residual_out,
|
||||
mean=mean,
|
||||
rstd=rstd,
|
||||
eps=eps,
|
||||
T=T,
|
||||
H=H,
|
||||
g_stride_n=g_stride_n,
|
||||
D=D,
|
||||
BD=BD,
|
||||
BT=BT,
|
||||
ACTIVATION=activation,
|
||||
IS_RMS_NORM=is_rms_norm,
|
||||
num_warps=8,
|
||||
launch_pdl=current_platform.is_arch_support_pdl(),
|
||||
)
|
||||
else:
|
||||
layer_norm_gated_fwd_kernel1[(T,)](
|
||||
x=x,
|
||||
g=g,
|
||||
y=y,
|
||||
w=weight,
|
||||
b=bias,
|
||||
residual=residual,
|
||||
residual_out=residual_out,
|
||||
mean=mean,
|
||||
rstd=rstd,
|
||||
eps=eps,
|
||||
D=D,
|
||||
BD=BD,
|
||||
ACTIVATION=activation,
|
||||
IS_RMS_NORM=is_rms_norm,
|
||||
num_warps=4,
|
||||
launch_pdl=current_platform.is_arch_support_pdl(),
|
||||
)
|
||||
# residual_out is None if residual is None and residual_dtype == input_dtype
|
||||
return y, mean, rstd, residual_out if residual_out is not None else x
|
||||
|
||||
|
||||
def rms_norm_gated(
|
||||
x: torch.Tensor,
|
||||
g: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
bias: torch.Tensor,
|
||||
activation: str = "swish",
|
||||
residual: torch.Tensor | None = None,
|
||||
prenorm: bool = False,
|
||||
residual_in_fp32: bool = False,
|
||||
eps: float = 1e-6,
|
||||
):
|
||||
x_shape_og = x.shape
|
||||
# reshape input data into 2D tensor
|
||||
x = x.contiguous().reshape(-1, x.shape[-1])
|
||||
D = x.shape[-1]
|
||||
# The tiled kernel supports row-strided gates; kernel1 does not.
|
||||
if D <= 512:
|
||||
H = 1 if g.ndim == 2 else g.shape[-2]
|
||||
g = g.view(-1, H, D)
|
||||
g_stride_n = g.stride(0)
|
||||
else:
|
||||
g = g.contiguous()
|
||||
H = 1
|
||||
g_stride_n = D
|
||||
if residual is not None:
|
||||
assert residual.shape == x_shape_og
|
||||
residual = residual.contiguous().reshape(-1, residual.shape[-1])
|
||||
residual_dtype = (
|
||||
residual.dtype
|
||||
if residual is not None
|
||||
else (torch.float if residual_in_fp32 else None)
|
||||
)
|
||||
y, _, _, residual_out = layer_norm_gated_fwd(
|
||||
x=x,
|
||||
g=g,
|
||||
weight=weight,
|
||||
bias=bias,
|
||||
activation=activation,
|
||||
eps=eps,
|
||||
residual=residual,
|
||||
residual_dtype=residual_dtype,
|
||||
is_rms_norm=True,
|
||||
H=H,
|
||||
g_stride_n=g_stride_n,
|
||||
)
|
||||
y = y.reshape(x_shape_og)
|
||||
return y if not prenorm else (y, residual_out.reshape(x_shape_og))
|
||||
|
||||
|
||||
@CustomOp.register("fused_rms_norm_gated")
|
||||
class FusedRMSNormGated(CustomOp):
|
||||
def __init__(
|
||||
self,
|
||||
hidden_size: int,
|
||||
elementwise_affine: bool = True,
|
||||
eps: float = 1e-5,
|
||||
activation: str = "swish",
|
||||
device: torch.device | None = None,
|
||||
dtype: torch.dtype | None = None,
|
||||
) -> None:
|
||||
factory_kwargs = {"device": device, "dtype": dtype}
|
||||
super().__init__()
|
||||
|
||||
self.hidden_size = hidden_size
|
||||
self.elementwise_affine = elementwise_affine
|
||||
self.eps = eps
|
||||
self.activation = activation
|
||||
|
||||
if self.activation not in ["swish", "silu", "sigmoid"]:
|
||||
raise ValueError(f"Unsupported activation: {self.activation}")
|
||||
|
||||
if elementwise_affine:
|
||||
self.weight = nn.Parameter(torch.empty(hidden_size, **factory_kwargs))
|
||||
else:
|
||||
self.register_parameter("weight", None)
|
||||
self.register_parameter("bias", None)
|
||||
|
||||
def forward_native(
|
||||
self,
|
||||
x: torch.Tensor,
|
||||
g: torch.Tensor,
|
||||
residual: torch.Tensor | None = None,
|
||||
prenorm: bool = False,
|
||||
residual_in_fp32: bool = False,
|
||||
) -> torch.Tensor:
|
||||
"""Decomposed PyTorch ops for torch.compile/inductor fusion."""
|
||||
# TODO(https://github.com/vllm-project/vllm/issues/36175): implement
|
||||
# native residual/prenorm path and unify with RMSNormGated.
|
||||
# For now, fall back to the triton kernel.
|
||||
if residual is not None or prenorm:
|
||||
return self.forward_cuda(x, g, residual, prenorm, residual_in_fp32)
|
||||
x_float = x.float()
|
||||
variance = x_float.pow(2).mean(dim=-1, keepdim=True)
|
||||
x_normed = x_float * torch.rsqrt(variance + self.eps)
|
||||
if self.weight is not None:
|
||||
x_normed = x_normed * self.weight.float()
|
||||
g_float = g.float()
|
||||
if self.activation in ("swish", "silu"):
|
||||
out = x_normed * g_float * torch.sigmoid(g_float)
|
||||
else: # sigmoid
|
||||
out = x_normed * torch.sigmoid(g_float)
|
||||
return out.to(x.dtype)
|
||||
|
||||
def forward_cuda(
|
||||
self,
|
||||
x: torch.Tensor,
|
||||
g: torch.Tensor,
|
||||
residual: torch.Tensor | None = None,
|
||||
prenorm: bool = False,
|
||||
residual_in_fp32: bool = False,
|
||||
) -> torch.Tensor:
|
||||
return rms_norm_gated(
|
||||
x,
|
||||
g,
|
||||
self.weight,
|
||||
self.bias,
|
||||
self.activation,
|
||||
residual=residual,
|
||||
eps=self.eps,
|
||||
prenorm=prenorm,
|
||||
residual_in_fp32=residual_in_fp32,
|
||||
)
|
||||
+63
-25
@@ -166,6 +166,8 @@ def layer_norm_gated_fwd_kernel(
|
||||
rstd, # pointer to the 1/std
|
||||
eps, # epsilon to avoid division by zero
|
||||
T, # number of rows in x
|
||||
H: tl.constexpr, # number of heads
|
||||
g_stride_n: tl.constexpr,
|
||||
D: tl.constexpr, # number of columns in x
|
||||
BT: tl.constexpr,
|
||||
BD: tl.constexpr,
|
||||
@@ -221,8 +223,13 @@ def layer_norm_gated_fwd_kernel(
|
||||
b_y = b_y + b_b[None, :]
|
||||
|
||||
# swish/sigmoid output gate
|
||||
p_g = tl.make_block_ptr(g, (T, D), (D, 1), (i_t * BT, 0), (BT, BD), (1, 0))
|
||||
b_g = tl.load(p_g, boundary_check=(0, 1)).to(tl.float32)
|
||||
o_t = i_t * BT + tl.arange(0, BT)
|
||||
o_g = (o_t // H) * g_stride_n + (o_t % H) * D
|
||||
b_g = tl.load(
|
||||
g + o_g[:, None] + o_d[None, :],
|
||||
mask=(o_t[:, None] < T) & m_d[None, :],
|
||||
other=0.0,
|
||||
).to(tl.float32)
|
||||
if ACTIVATION == "swish" or ACTIVATION == "silu":
|
||||
b_y = b_y * b_g * tl.sigmoid(b_g)
|
||||
elif ACTIVATION == "sigmoid":
|
||||
@@ -320,10 +327,15 @@ def layer_norm_gated_fwd(
|
||||
out_dtype: torch.dtype = None,
|
||||
residual_dtype: torch.dtype = None,
|
||||
is_rms_norm: bool = False,
|
||||
H: int = 1,
|
||||
g_stride_n: int | None = None,
|
||||
):
|
||||
if residual is not None:
|
||||
residual_dtype = residual.dtype
|
||||
T, D = x.shape
|
||||
if g_stride_n is None:
|
||||
g_stride_n = D
|
||||
assert T % H == 0
|
||||
if residual is not None:
|
||||
assert residual.shape == (T, D)
|
||||
if weight is not None:
|
||||
@@ -349,10 +361,8 @@ def layer_norm_gated_fwd(
|
||||
BD = min(MAX_FUSED_SIZE, next_power_of_2(D))
|
||||
if D > BD:
|
||||
raise RuntimeError("This layer norm doesn't support feature dim >= 64KB.")
|
||||
# heuristics for number of warps
|
||||
|
||||
if D <= 512:
|
||||
BT = 32
|
||||
BT = 16
|
||||
layer_norm_gated_fwd_kernel[(cdiv(T, BT),)](
|
||||
x=x,
|
||||
g=g,
|
||||
@@ -365,12 +375,14 @@ def layer_norm_gated_fwd(
|
||||
rstd=rstd,
|
||||
eps=eps,
|
||||
T=T,
|
||||
H=H,
|
||||
g_stride_n=g_stride_n,
|
||||
D=D,
|
||||
BD=BD,
|
||||
BT=BT,
|
||||
ACTIVATION=activation,
|
||||
IS_RMS_NORM=is_rms_norm,
|
||||
num_warps=4,
|
||||
num_warps=8,
|
||||
)
|
||||
else:
|
||||
layer_norm_gated_fwd_kernel1[(T,)](
|
||||
@@ -408,7 +420,16 @@ def rms_norm_gated(
|
||||
x_shape_og = x.shape
|
||||
# reshape input data into 2D tensor
|
||||
x = x.contiguous().reshape(-1, x.shape[-1])
|
||||
g = g.contiguous().reshape(-1, g.shape[-1])
|
||||
D = x.shape[-1]
|
||||
# The tiled kernel supports row-strided gates; kernel1 does not.
|
||||
if D <= 512:
|
||||
H = 1 if g.ndim == 2 else g.shape[-2]
|
||||
g = g.view(-1, H, D)
|
||||
g_stride_n = g.stride(0)
|
||||
else:
|
||||
g = g.contiguous()
|
||||
H = 1
|
||||
g_stride_n = D
|
||||
if residual is not None:
|
||||
assert residual.shape == x_shape_og
|
||||
residual = residual.contiguous().reshape(-1, residual.shape[-1])
|
||||
@@ -427,6 +448,8 @@ def rms_norm_gated(
|
||||
residual=residual,
|
||||
residual_dtype=residual_dtype,
|
||||
is_rms_norm=True,
|
||||
H=H,
|
||||
g_stride_n=g_stride_n,
|
||||
)
|
||||
y = y.reshape(x_shape_og)
|
||||
return y if not prenorm else (y, residual_out.reshape(x_shape_og))
|
||||
@@ -1187,6 +1210,7 @@ def kda_gate_cumsum_fwd_kernel(
|
||||
cu_seqlens,
|
||||
chunk_indices,
|
||||
cumsum_scale,
|
||||
lower_bound,
|
||||
beta,
|
||||
threshold,
|
||||
T,
|
||||
@@ -1196,6 +1220,7 @@ def kda_gate_cumsum_fwd_kernel(
|
||||
BD: tl.constexpr,
|
||||
HAS_BIAS: tl.constexpr,
|
||||
IS_VARLEN: tl.constexpr,
|
||||
USE_LOWER_BOUND: tl.constexpr,
|
||||
):
|
||||
i_d, i_t, i_bh = tl.program_id(0), tl.program_id(1), tl.program_id(2)
|
||||
i_b, i_h = i_bh // H, i_bh % H
|
||||
@@ -1235,14 +1260,17 @@ def kda_gate_cumsum_fwd_kernel(
|
||||
b_bias = tl.load(g_bias + i_h * D + o_d, mask=o_d < D, other=0.0).to(tl.float32)
|
||||
b_g = b_g + b_bias[None, :]
|
||||
|
||||
b_a = -tl.exp(tl.load(A + i_h).to(tl.float32))
|
||||
b_g_scaled = b_g * beta
|
||||
b_softplus = tl.where(
|
||||
b_g_scaled > threshold,
|
||||
b_g,
|
||||
(1.0 / beta) * log(1.0 + tl.exp(b_g_scaled)),
|
||||
)
|
||||
b_gate = b_a * b_softplus
|
||||
b_a = tl.exp(tl.load(A + i_h).to(tl.float32))
|
||||
if USE_LOWER_BOUND:
|
||||
b_gate = lower_bound * tl.sigmoid(b_a * b_g)
|
||||
else:
|
||||
b_g_scaled = b_g * beta
|
||||
b_softplus = tl.where(
|
||||
b_g_scaled > threshold,
|
||||
b_g,
|
||||
(1.0 / beta) * log(1.0 + tl.exp(b_g_scaled)),
|
||||
)
|
||||
b_gate = -b_a * b_softplus
|
||||
|
||||
# Out-of-bounds rows (load returns 0, but softplus/bias can still make
|
||||
# b_gate non-zero) participate in the dot product. They only contribute to
|
||||
@@ -1260,6 +1288,7 @@ def fused_kda_gate_chunk_cumsum(
|
||||
g_bias: torch.Tensor | None = None,
|
||||
beta: float = 1.0,
|
||||
threshold: float = 20.0,
|
||||
lower_bound: float | None = None,
|
||||
cu_seqlens: torch.Tensor | None = None,
|
||||
chunk_indices: torch.Tensor | None = None,
|
||||
chunk_size: int = FLA_CHUNK_SIZE,
|
||||
@@ -1293,16 +1322,17 @@ def fused_kda_gate_chunk_cumsum(
|
||||
# exp2-based kernels reproduce exp(g). Keep this in sync with the
|
||||
# `use_exp2=True` path in `_chunk_kda_fwd_with_cumulative_g`.
|
||||
cumsum_scale=RCP_LN2,
|
||||
lower_bound=lower_bound or 0.0,
|
||||
beta=beta,
|
||||
threshold=threshold,
|
||||
T=T,
|
||||
H=H,
|
||||
D=D,
|
||||
BT=chunk_size,
|
||||
USE_LOWER_BOUND=lower_bound is not None,
|
||||
)
|
||||
return y
|
||||
|
||||
|
||||
def _chunk_kda_fwd_with_cumulative_g(
|
||||
q: torch.Tensor,
|
||||
k: torch.Tensor,
|
||||
@@ -1424,6 +1454,7 @@ def chunk_kda_with_fused_gate_fwd(
|
||||
scale: float,
|
||||
initial_state: torch.Tensor,
|
||||
output_final_state: bool,
|
||||
lower_bound: float | None = None,
|
||||
cu_seqlens: torch.Tensor | None = None,
|
||||
):
|
||||
chunk_size = FLA_CHUNK_SIZE
|
||||
@@ -1439,6 +1470,7 @@ def chunk_kda_with_fused_gate_fwd(
|
||||
cu_seqlens=cu_seqlens,
|
||||
chunk_indices=chunk_indices,
|
||||
chunk_size=chunk_size,
|
||||
lower_bound=lower_bound,
|
||||
)
|
||||
return _chunk_kda_fwd_with_cumulative_g(
|
||||
q=q,
|
||||
@@ -1500,6 +1532,7 @@ def chunk_kda_with_fused_gate(
|
||||
scale: float | None = None,
|
||||
initial_state: torch.Tensor | None = None,
|
||||
output_final_state: bool = False,
|
||||
lower_bound: float | None = None,
|
||||
use_qk_l2norm_in_kernel: bool = False,
|
||||
cu_seqlens: torch.Tensor | None = None,
|
||||
**kwargs,
|
||||
@@ -1523,6 +1556,7 @@ def chunk_kda_with_fused_gate(
|
||||
scale=scale,
|
||||
initial_state=initial_state.contiguous() if initial_state is not None else None,
|
||||
output_final_state=output_final_state,
|
||||
lower_bound=lower_bound,
|
||||
cu_seqlens=cu_seqlens,
|
||||
)
|
||||
return o, final_state
|
||||
@@ -1543,6 +1577,7 @@ def kda_gate_fwd_kernel(
|
||||
A,
|
||||
y,
|
||||
g_bias,
|
||||
lower_bound,
|
||||
beta: tl.constexpr,
|
||||
threshold: tl.constexpr,
|
||||
T,
|
||||
@@ -1551,12 +1586,12 @@ def kda_gate_fwd_kernel(
|
||||
BT: tl.constexpr,
|
||||
BD: tl.constexpr,
|
||||
HAS_BIAS: tl.constexpr,
|
||||
USE_LOWER_BOUND: tl.constexpr,
|
||||
):
|
||||
i_t, i_h = tl.program_id(0), tl.program_id(1)
|
||||
n_t = i_t * BT
|
||||
|
||||
b_a = tl.load(A + i_h).to(tl.float32)
|
||||
b_a = -tl.exp(b_a)
|
||||
b_a = tl.exp(tl.load(A + i_h).to(tl.float32))
|
||||
|
||||
stride_row = H * D
|
||||
stride_col = 1
|
||||
@@ -1589,13 +1624,13 @@ def kda_gate_fwd_kernel(
|
||||
)
|
||||
b_g = b_g + b_bias[None, :]
|
||||
|
||||
# softplus(x, beta) = (1/beta) * log(1 + exp(beta * x))
|
||||
# When beta * x > threshold, use linear approximation x
|
||||
# Use threshold to switch to linear when beta*x > threshold
|
||||
g_scaled = b_g * beta
|
||||
use_linear = g_scaled > threshold
|
||||
sp = tl.where(use_linear, b_g, (1.0 / beta) * log(1.0 + tl.exp(g_scaled)))
|
||||
b_y = b_a * sp
|
||||
if USE_LOWER_BOUND:
|
||||
b_y = lower_bound * tl.sigmoid(b_a * b_g)
|
||||
else:
|
||||
g_scaled = b_g * beta
|
||||
use_linear = g_scaled > threshold
|
||||
sp = tl.where(use_linear, b_g, (1.0 / beta) * log(1.0 + tl.exp(g_scaled)))
|
||||
b_y = -b_a * sp
|
||||
|
||||
tl.store(y_ptr, b_y.to(y.dtype.element_ty), boundary_check=(0, 1))
|
||||
|
||||
@@ -1607,6 +1642,7 @@ def fused_kda_gate(
|
||||
g_bias: torch.Tensor | None = None,
|
||||
beta: float = 1.0,
|
||||
threshold: float = 20.0,
|
||||
lower_bound: float | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Forward pass for KDA gate:
|
||||
@@ -1634,6 +1670,7 @@ def fused_kda_gate(
|
||||
A,
|
||||
y,
|
||||
g_bias,
|
||||
lower_bound or 0.0,
|
||||
beta,
|
||||
threshold,
|
||||
T,
|
||||
@@ -1641,6 +1678,7 @@ def fused_kda_gate(
|
||||
head_k_dim,
|
||||
BD=next_power_of_2(head_k_dim),
|
||||
HAS_BIAS=g_bias is not None,
|
||||
USE_LOWER_BOUND=lower_bound is not None,
|
||||
)
|
||||
|
||||
y = y.view(*orig_shape, H, head_k_dim)
|
||||
|
||||
@@ -0,0 +1,139 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Kimi-K3 multimodal configuration."""
|
||||
|
||||
from transformers.configuration_utils import PretrainedConfig
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.transformers_utils.configs.kimi_linear import KimiLinearConfig
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class KimiK3VisionConfig(PretrainedConfig):
|
||||
model_type = "kimi_k3_vision"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
patch_size: int = 14,
|
||||
init_pos_emb_height: int = 64,
|
||||
init_pos_emb_width: int = 64,
|
||||
init_pos_emb_time: int = 4,
|
||||
pos_emb_type: str = "divided_fixed",
|
||||
vt_num_attention_heads: int = 12,
|
||||
vt_num_hidden_layers: int = 27,
|
||||
vt_hidden_size: int = 1024,
|
||||
vt_intermediate_size: int = 4096,
|
||||
merge_kernel_size: tuple[int, int] = (2, 2),
|
||||
video_attn_type: str = "spatial_temporal",
|
||||
merge_type: str = "sd2_tpool",
|
||||
_attn_implementation: str = "flash_attention_2",
|
||||
mm_projector_type: str = "patchmergerv2",
|
||||
mm_hidden_size: int | None = None,
|
||||
projector_hidden_act: str = "gelu",
|
||||
projector_ln_eps: float = 1e-5,
|
||||
qkv_hidden_size: int = 1536,
|
||||
norm_type: str = "rmsnorm",
|
||||
attn_bias: bool = False,
|
||||
patch_embed_proj_bias: bool = False,
|
||||
mlp_type: str = "mlp2",
|
||||
linear_bias: bool = False,
|
||||
activation_func: str = "gelu_pytorch_tanh",
|
||||
pos_emb_interpolation_mode: str = "bilinear",
|
||||
text_hidden_size: int = 2304,
|
||||
**kwargs,
|
||||
):
|
||||
super().__init__(**kwargs)
|
||||
|
||||
self.patch_size = patch_size
|
||||
self.init_pos_emb_height = init_pos_emb_height
|
||||
self.init_pos_emb_width = init_pos_emb_width
|
||||
self.init_pos_emb_time = init_pos_emb_time
|
||||
self.pos_emb_type = pos_emb_type
|
||||
self.vt_num_attention_heads = vt_num_attention_heads
|
||||
self.vt_num_hidden_layers = vt_num_hidden_layers
|
||||
self.vt_hidden_size = vt_hidden_size
|
||||
self.vt_intermediate_size = vt_intermediate_size
|
||||
self.merge_kernel_size = tuple(merge_kernel_size)
|
||||
self.video_attn_type = video_attn_type
|
||||
self.merge_type = merge_type
|
||||
self._attn_implementation = _attn_implementation
|
||||
|
||||
self.mm_projector_type = mm_projector_type
|
||||
self.mm_hidden_size = (
|
||||
mm_hidden_size if mm_hidden_size is not None else vt_hidden_size
|
||||
)
|
||||
self.projector_hidden_act = projector_hidden_act
|
||||
self.projector_ln_eps = projector_ln_eps
|
||||
self.text_hidden_size = text_hidden_size
|
||||
|
||||
self.qkv_hidden_size = qkv_hidden_size
|
||||
self.norm_type = norm_type
|
||||
self.attn_bias = attn_bias
|
||||
self.patch_embed_proj_bias = patch_embed_proj_bias
|
||||
self.mlp_type = mlp_type
|
||||
self.linear_bias = linear_bias
|
||||
self.activation_func = activation_func
|
||||
self.pos_emb_interpolation_mode = pos_emb_interpolation_mode
|
||||
|
||||
# Aliases consumed by the Kimi-K2.5 vision implementation.
|
||||
self.num_attention_heads = vt_num_attention_heads
|
||||
self.num_hidden_layers = vt_num_hidden_layers
|
||||
self.hidden_size = vt_hidden_size
|
||||
self.intermediate_size = vt_intermediate_size
|
||||
|
||||
|
||||
class KimiK3Config(PretrainedConfig):
|
||||
model_type = "kimi_k3"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
text_config: dict | KimiLinearConfig | None = None,
|
||||
vision_config: dict | KimiK3VisionConfig | None = None,
|
||||
ignore_index: int = -100,
|
||||
media_placeholder_token_id: int = 163605,
|
||||
pad_token_id: int = 0,
|
||||
image_placeholder: str = "<|kimi_image_placeholder|>",
|
||||
**kwargs,
|
||||
):
|
||||
if text_config is None:
|
||||
self.text_config = KimiLinearConfig()
|
||||
elif isinstance(text_config, dict):
|
||||
self.text_config = KimiLinearConfig(**text_config)
|
||||
else:
|
||||
self.text_config = text_config
|
||||
|
||||
if vision_config is None:
|
||||
self.vision_config = KimiK3VisionConfig()
|
||||
elif isinstance(vision_config, dict):
|
||||
self.vision_config = KimiK3VisionConfig(**vision_config)
|
||||
else:
|
||||
self.vision_config = vision_config
|
||||
|
||||
# K3's vision projector output must match the text model's hidden size.
|
||||
# Override any value provided in vision_config for safety.
|
||||
if self.vision_config.text_hidden_size != self.text_config.hidden_size:
|
||||
logger.info(
|
||||
"Overriding vision_config.text_hidden_size from %s to %s "
|
||||
"to match text_config.hidden_size",
|
||||
self.vision_config.text_hidden_size,
|
||||
self.text_config.hidden_size,
|
||||
)
|
||||
self.vision_config.text_hidden_size = self.text_config.hidden_size
|
||||
|
||||
self.ignore_index = ignore_index
|
||||
self.media_placeholder_token_id = media_placeholder_token_id
|
||||
self.image_placeholder = image_placeholder
|
||||
|
||||
if getattr(self.text_config, "quantization_config", None) is not None:
|
||||
self.quantization_config = self.text_config.quantization_config
|
||||
|
||||
super().__init__(pad_token_id=pad_token_id, **kwargs)
|
||||
|
||||
@property
|
||||
def hidden_size(self) -> int:
|
||||
return self.text_config.hidden_size
|
||||
|
||||
@property
|
||||
def vocab_size(self) -> int:
|
||||
return self.text_config.vocab_size
|
||||
@@ -0,0 +1,61 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from transformers import BaseImageProcessor, BatchFeature, TensorType
|
||||
from transformers.processing_utils import ProcessorMixin
|
||||
|
||||
from vllm.tokenizers.hf import HfTokenizer
|
||||
|
||||
|
||||
class KimiK3Processor(ProcessorMixin):
|
||||
"""HF-style processor wrapper for the image-only Kimi-K3 model.
|
||||
|
||||
K3 exposes the standard ``image`` modality, so vLLM calls this processor
|
||||
with ``images=[PIL, ...]``. The underlying checkpoint image processor
|
||||
(``KimiK3VisionProcessor``) works on ``{"type": "image", "image": PIL}``
|
||||
media dicts, so this wrapper adapts bare PIL images into that shape before
|
||||
delegating to ``preprocess``.
|
||||
|
||||
Text is only tokenized here; the single ``<|kimi_image_placeholder|>``
|
||||
token per image is expanded into the resolution-aware media block by the
|
||||
model's ``_get_prompt_updates`` on the vLLM side.
|
||||
"""
|
||||
|
||||
attributes = ["image_processor", "tokenizer"]
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
image_processor: BaseImageProcessor,
|
||||
tokenizer: HfTokenizer,
|
||||
) -> None:
|
||||
self.image_processor = image_processor
|
||||
self.tokenizer = tokenizer
|
||||
|
||||
def __call__(
|
||||
self,
|
||||
text: str | list[str] | None = None,
|
||||
images: object | list[object] | None = None,
|
||||
return_tensors: str | TensorType | None = None,
|
||||
**kwargs,
|
||||
) -> BatchFeature:
|
||||
if images is not None:
|
||||
if not isinstance(images, list):
|
||||
images = [images]
|
||||
medias = [{"type": "image", "image": image} for image in images]
|
||||
mm_inputs = self.image_processor.preprocess(
|
||||
medias,
|
||||
return_tensors=return_tensors,
|
||||
)
|
||||
else:
|
||||
mm_inputs = {}
|
||||
|
||||
if text is not None:
|
||||
if not isinstance(text, list):
|
||||
text = [text]
|
||||
text_inputs = self.tokenizer(text)
|
||||
else:
|
||||
text_inputs = {}
|
||||
|
||||
return BatchFeature(
|
||||
data={**text_inputs, **mm_inputs},
|
||||
tensor_type=return_tensors,
|
||||
)
|
||||
@@ -142,6 +142,15 @@ def merge_attn_states(
|
||||
# attention backend.
|
||||
prefix_head_stride = prefix_output.stride(1)
|
||||
output_head_stride = output.stride(1)
|
||||
# lse tensors are [NUM_HEADS, NUM_TOKENS] but may be non-contiguous views
|
||||
# (e.g. a transpose of a backend's [NUM_TOKENS, NUM_HEADS] output), so index
|
||||
# them by their actual strides rather than assuming a contiguous layout.
|
||||
prefix_lse_head_stride = prefix_lse.stride(0)
|
||||
prefix_lse_token_stride = prefix_lse.stride(1)
|
||||
suffix_lse_head_stride = suffix_lse.stride(0)
|
||||
suffix_lse_token_stride = suffix_lse.stride(1)
|
||||
output_lse_head_stride = output_lse.stride(0) if output_lse is not None else 0
|
||||
output_lse_token_stride = output_lse.stride(1) if output_lse is not None else 0
|
||||
|
||||
# If prefill_tokens_with_context is None, all tokens should use prefix context
|
||||
if prefill_tokens_with_context is None:
|
||||
@@ -157,6 +166,12 @@ def merge_attn_states(
|
||||
suffix_lse,
|
||||
prefix_head_stride,
|
||||
output_head_stride,
|
||||
prefix_lse_head_stride,
|
||||
prefix_lse_token_stride,
|
||||
suffix_lse_head_stride,
|
||||
suffix_lse_token_stride,
|
||||
output_lse_head_stride,
|
||||
output_lse_token_stride,
|
||||
output_scale,
|
||||
prefill_tokens_with_context,
|
||||
head_size,
|
||||
@@ -176,6 +191,12 @@ def merge_attn_states_kernel(
|
||||
suffix_lse, # [NUM_HEADS, NUM_TOKENS]
|
||||
prefix_head_stride,
|
||||
output_head_stride,
|
||||
prefix_lse_head_stride,
|
||||
prefix_lse_token_stride,
|
||||
suffix_lse_head_stride,
|
||||
suffix_lse_token_stride,
|
||||
output_lse_head_stride,
|
||||
output_lse_token_stride,
|
||||
output_scale, # scale tensor or None
|
||||
prefill_tokens_with_context,
|
||||
HEAD_SIZE: tl.constexpr,
|
||||
@@ -186,7 +207,6 @@ def merge_attn_states_kernel(
|
||||
FP8_MAX: tl.constexpr = float8_info.max,
|
||||
):
|
||||
token_idx = tl.program_id(0)
|
||||
num_tokens = tl.num_programs(0)
|
||||
head_idx = tl.program_id(1)
|
||||
num_heads = tl.num_programs(1)
|
||||
|
||||
@@ -198,9 +218,18 @@ def merge_attn_states_kernel(
|
||||
# For tokens without context (token_idx >= prefill_tokens_with_context),
|
||||
# directly copy from suffix_output
|
||||
if not prefix_mask:
|
||||
s_lse = tl.load(suffix_lse + head_idx * num_tokens + token_idx)
|
||||
s_lse = tl.load(
|
||||
suffix_lse
|
||||
+ head_idx * suffix_lse_head_stride
|
||||
+ token_idx * suffix_lse_token_stride
|
||||
)
|
||||
if OUTPUT_LSE:
|
||||
tl.store(output_lse + head_idx * num_tokens + token_idx, s_lse)
|
||||
tl.store(
|
||||
output_lse
|
||||
+ head_idx * output_lse_head_stride
|
||||
+ token_idx * output_lse_token_stride,
|
||||
s_lse,
|
||||
)
|
||||
|
||||
s_out = tl.load(
|
||||
suffix_output
|
||||
@@ -227,8 +256,16 @@ def merge_attn_states_kernel(
|
||||
|
||||
# For tokens with context (token_idx < prefill_tokens_with_context),
|
||||
# perform normal merge operation
|
||||
p_lse = tl.load(prefix_lse + head_idx * num_tokens + token_idx)
|
||||
s_lse = tl.load(suffix_lse + head_idx * num_tokens + token_idx)
|
||||
p_lse = tl.load(
|
||||
prefix_lse
|
||||
+ head_idx * prefix_lse_head_stride
|
||||
+ token_idx * prefix_lse_token_stride
|
||||
)
|
||||
s_lse = tl.load(
|
||||
suffix_lse
|
||||
+ head_idx * suffix_lse_head_stride
|
||||
+ token_idx * suffix_lse_token_stride
|
||||
)
|
||||
|
||||
# FA2 and FA3 have different behavior for when the sum-exp is 0, this namely
|
||||
# arises with 0 len seqlens. FA3 returns -inf here while FA2 returns inf.
|
||||
@@ -251,7 +288,12 @@ def merge_attn_states_kernel(
|
||||
# Both sides empty (max_lse == -inf) => undefined merge; keep -inf so
|
||||
# downstream merges continue to treat the token as empty.
|
||||
out_lse = tl.where(max_lse == float("-inf"), float("-inf"), out_lse)
|
||||
tl.store(output_lse + head_idx * num_tokens + token_idx, out_lse)
|
||||
tl.store(
|
||||
output_lse
|
||||
+ head_idx * output_lse_head_stride
|
||||
+ token_idx * output_lse_token_stride,
|
||||
out_lse,
|
||||
)
|
||||
|
||||
p_out = tl.load(
|
||||
prefix_output
|
||||
|
||||
Reference in New Issue
Block a user