[Model] Add Kimi K3 support: model files and kernels [1/N] (#50089)

Signed-off-by: zjy0516 <riverclouds.zhu@qq.com>
Signed-off-by: Isotr0py <Isotr0py@outlook.com>
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
Signed-off-by: Yongye Zhu <zyy1102000@gmail.com>
Co-authored-by: Isotr0py <Isotr0py@outlook.com>
Co-authored-by: Thien Tran <gau.nernst@yahoo.com.sg>
Co-authored-by: Bugen Zhao <i@bugenzhao.com>
Co-authored-by: Yongye Zhu <zyy1102000@gmail.com>
Co-authored-by: Ziming Huang <zelda.huanghuang@gmail.com>
Co-authored-by: Roger Wang <hey@rogerw.io>
Co-authored-by: Isotr0py <mozf@inferact.ai>
Co-authored-by: aoshen02 <aoshen02@users.noreply.github.com>
Co-authored-by: Woosuk Kwon <woosuk@inferact.ai>
Co-authored-by: Jee Jee Li <jeejeelee@inferact.ai>
Co-authored-by: aoshen02 <aoshen@inferact.ai>
Co-authored-by: Summer Yang <girasoleyang@gmail.com>
Co-authored-by: Kevin H. Luu <khluu000@gmail.com>
Co-authored-by: Bowen Wang <abmfy@icloud.com>
Co-authored-by: gnovack <novackgm@gmail.com>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai>
Co-authored-by: xiaozhoupy <peiyuanzhou1994@gmail.com>
Co-authored-by: Roy Wang <yasong.wang@inferact.ai>
Co-authored-by: Jeff (Junze) Ma <93145857+majunze2001@users.noreply.github.com>
Co-authored-by: Codex <codex@openai.com>
This commit is contained in:
Jiangyun Zhu
2026-07-29 14:10:58 +08:00
committed by GitHub
co-authored by Isotr0py Thien Tran Bugen Zhao Yongye Zhu Ziming Huang Roger Wang Isotr0py aoshen02 Woosuk Kwon Jee Jee Li aoshen02 Summer Yang Kevin H. Luu Bowen Wang gnovack Nick Hill Yifan Qiao xiaozhoupy Roy Wang Jeff Ma Codex
parent 6f00a1ae3b
commit 7c6729b769
109 changed files with 28028 additions and 792 deletions
@@ -0,0 +1,180 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from __future__ import annotations
import cutlass
import cutlass.cute as cute
from cuda.bindings.driver import CUstream
from cutlass import const_expr
class CuteSkinnyGemm:
"""Shape-dynamic low-latency GEMM for small token counts.
Computes ``C[M, N] = A[M, K] @ B[N, K].T + residual`` with BF16 or FP16
inputs, FP32 accumulators, and an output matching the input dtype. The
residual term is optional and is added before the output conversion. N and
K are runtime values. The tiny M dimension is fully unrolled alongside a
small set of tuning parameters.
"""
def __init__(
self,
*,
element_type,
num_rows: int,
block_size: int,
outputs_per_block: int,
vector_width: int = 8,
k_unroll: int = 1,
has_residual: bool = False,
use_pdl: bool = False,
) -> None:
if block_size % cute.arch.WARP_SIZE != 0:
raise ValueError("block_size must be a multiple of the warp size")
self.element_type = element_type
self.num_rows = num_rows
self.block_size = block_size
self.outputs_per_block = outputs_per_block
self.vector_width = vector_width
self.k_unroll = k_unroll
self.has_residual = has_residual
self.use_pdl = use_pdl
self.num_warps = block_size // cute.arch.WARP_SIZE
@cute.jit
def __call__(
self,
gA: cute.Tensor,
gB: cute.Tensor,
gResidual: cute.Tensor,
gC: cute.Tensor,
stream: CUstream,
) -> None:
n = cute.size(gB, mode=[0])
k = cute.size(gA, mode=[1])
copy_a = cute.make_copy_atom(
cute.nvgpu.CopyG2ROp(),
self.element_type,
num_bits_per_copy=self.vector_width * self.element_type.width,
load_cache_mode=cute.nvgpu.LoadCacheMode.ALWAYS,
)
copy_b = cute.make_copy_atom(
cute.nvgpu.CopyG2ROp(),
self.element_type,
num_bits_per_copy=self.vector_width * self.element_type.width,
load_cache_mode=cute.nvgpu.LoadCacheMode.STREAMING,
)
self.kernel(gA, gB, gResidual, gC, k, copy_a, copy_b).launch(
grid=[cute.ceil_div(n, self.outputs_per_block), 1, 1],
block=[self.block_size, 1, 1],
smem=self.num_rows * self.outputs_per_block * self.num_warps * 4,
stream=stream,
use_pdl=self.use_pdl,
min_blocks_per_mp=1,
)
@cute.kernel
def kernel(
self,
gA: cute.Tensor,
gB: cute.Tensor,
gResidual: cute.Tensor,
gC: cute.Tensor,
k_extent: cutlass.Int32,
copy_a: cute.CopyAtom,
copy_b: cute.CopyAtom,
) -> None:
tidx, _, _ = cute.arch.thread_idx()
block_idx, _, _ = cute.arch.block_idx()
warp_idx = cute.arch.warp_idx()
num_rows: cutlass.Constexpr = self.num_rows
outputs_per_block: cutlass.Constexpr = self.outputs_per_block
vector_width: cutlass.Constexpr = self.vector_width
block_size: cutlass.Constexpr = self.block_size
num_warps: cutlass.Constexpr = self.num_warps
acc_layout = cute.make_layout(
(num_rows, outputs_per_block), stride=(outputs_per_block, 1)
)
acc = cute.make_rmem_tensor(acc_layout, cutlass.Float32)
acc.fill(0.0)
if const_expr(self.use_pdl):
cute.arch.griddepcontrol_wait()
n_base = block_idx * outputs_per_block
k_tile_size: cutlass.Constexpr = block_size * vector_width
num_k_tiles = k_extent // k_tile_size
gA_vec = cute.logical_divide(gA, (None, vector_width))
gB_vec = cute.logical_divide(gB, (None, vector_width))
# Layout after both divides is (M/N, K_TILE, K_LANE, K_VEC).
tA_all = cute.logical_divide(gA_vec, (None, (None, block_size)))
tB_all = cute.logical_divide(gB_vec, (None, (None, block_size)))
tA = tA_all[None, (None, (tidx, None))]
a_regs = cute.make_rmem_tensor(
cute.make_layout((num_rows, vector_width), stride=(vector_width, 1)),
self.element_type,
)
b_regs = cute.make_rmem_tensor(
cute.make_layout(
(outputs_per_block, vector_width), stride=(vector_width, 1)
),
self.element_type,
)
for k_tile in cutlass.range(num_k_tiles, unroll=self.k_unroll):
for mi in cutlass.range_constexpr(num_rows):
cute.copy(copy_a, tA[mi, None, k_tile], a_regs[mi, None])
for ni in cutlass.range_constexpr(outputs_per_block):
n_idx = n_base + ni
tB = tB_all[n_idx, (None, (tidx, None))]
cute.copy(copy_b, tB[None, k_tile], b_regs[ni, None])
for vi in cutlass.range_constexpr(vector_width):
for mi in cutlass.range_constexpr(num_rows):
for ni in cutlass.range_constexpr(outputs_per_block):
acc[mi, ni] = acc[mi, ni] + a_regs[mi, vi].to(
cutlass.Float32
) * b_regs[ni, vi].to(cutlass.Float32)
for mi in cutlass.range_constexpr(num_rows):
for ni in cutlass.range_constexpr(outputs_per_block):
acc[mi, ni] = cute.arch.warp_reduction_sum(acc[mi, ni])
smem_layout = cute.make_layout(
(num_rows, outputs_per_block, num_warps),
stride=(outputs_per_block * num_warps, num_warps, 1),
)
smem = cutlass.utils.SmemAllocator()
partials = smem.allocate_tensor(cutlass.Float32, smem_layout, byte_alignment=16)
with cute.arch.elect_one():
for mi in cutlass.range_constexpr(num_rows):
for ni in cutlass.range_constexpr(outputs_per_block):
partials[mi, ni, warp_idx] = acc[mi, ni]
cute.arch.sync_threads()
if tidx == 0:
for mi in cutlass.range_constexpr(num_rows):
for ni in cutlass.range_constexpr(outputs_per_block):
n_idx = n_base + ni
total = (
partials[mi, ni, None]
.load()
.reduce(
cute.ReductionOp.ADD,
init_val=cutlass.Float32(0.0),
reduction_profile=0,
)
)
if const_expr(self.has_residual):
total += gResidual[mi, n_idx].to(cutlass.Float32)
gC[mi, n_idx] = cutlass.Float32(total).to(self.element_type)
if const_expr(self.use_pdl):
cute.arch.griddepcontrol_launch_dependents()
@@ -0,0 +1,252 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from __future__ import annotations
import logging
from collections.abc import Iterable
from dataclasses import dataclass
from functools import partial
from typing import Any
import torch
logger = logging.getLogger(__name__)
_cutedsl_available: bool | None = None
@dataclass(frozen=True, slots=True)
class SkinnyGemmConfig:
num_rows: int
block_size: int
outputs_per_block: int
k_unroll: int = 1
vector_width: int = 8
class ShapeDynamicSkinnyGemm:
def __init__(self) -> None:
self._compiled: dict[tuple[torch.dtype, SkinnyGemmConfig, bool], Any] = {}
self._warmup_configs: set[tuple[torch.dtype, SkinnyGemmConfig, bool]] = set()
self._warmup_registered = False
@staticmethod
def is_available() -> bool:
global _cutedsl_available
if _cutedsl_available is not None:
return _cutedsl_available
try:
import cutlass # noqa: F401
import cutlass.cute # noqa: F401
_cutedsl_available = True
except ImportError:
_cutedsl_available = False
logger.info("cuteDSL is not available; skinny GEMM is disabled")
return _cutedsl_available
@staticmethod
def _config(m: int, n: int, k: int) -> SkinnyGemmConfig:
num_rows = m
wide_block = 224
if m == 1 and k >= 7168 and k % (wide_block * 8) == 0:
if n % 3 == 0:
k_unroll = 2 if n <= 2304 else 4
return SkinnyGemmConfig(num_rows, wide_block, 3, k_unroll)
if 2304 < n < 4096 and n % 2 == 0:
return SkinnyGemmConfig(num_rows, wide_block, 2, k_unroll=4)
if k <= 2048 or k % (128 * 8) != 0:
outputs_per_block = 2 if k <= 2048 else 4
if n % outputs_per_block:
outputs_per_block = 1
if k % (64 * 8) == 0:
return SkinnyGemmConfig(num_rows, 64, outputs_per_block, 2)
if k % (32 * 8) == 0:
return SkinnyGemmConfig(num_rows, 32, outputs_per_block, 2)
return SkinnyGemmConfig(num_rows, 32, outputs_per_block, 2, vector_width=4)
block_size = 64 if 4096 <= n < 8192 else 128
outputs_per_block = 1 if m == 1 and n <= 2304 else 2
if n % outputs_per_block:
outputs_per_block = 1
k_unroll = 2 if n <= 2304 or n >= 16384 else 1
return SkinnyGemmConfig(
num_rows,
block_size,
outputs_per_block,
k_unroll=k_unroll,
)
@staticmethod
def _cutlass_dtype(dtype: torch.dtype):
from cutlass import BFloat16, Float16
return BFloat16 if dtype == torch.bfloat16 else Float16
@staticmethod
def _stream():
from cuda.bindings.driver import CUstream
from vllm.utils.torch_utils import current_stream
return CUstream(current_stream().cuda_stream)
@staticmethod
def _use_pdl() -> bool:
from vllm.platforms import current_platform
return current_platform.is_arch_support_pdl()
def _compile(
self,
dtype: torch.dtype,
config: SkinnyGemmConfig,
has_residual: bool,
) -> None:
import cutlass.cute as cute
from quack.compile_utils import make_fake_tensor
from ._skinny_gemm import CuteSkinnyGemm
element_type = self._cutlass_dtype(dtype)
n = cute.sym_int(divisibility=config.outputs_per_block)
k = cute.sym_int(divisibility=config.block_size * config.vector_width)
a = make_fake_tensor(
element_type,
(config.num_rows, k),
divisibility=config.vector_width,
)
b = make_fake_tensor(
element_type,
(n, k),
divisibility=config.vector_width,
)
c = make_fake_tensor(element_type, (config.num_rows, n), divisibility=1)
residual = make_fake_tensor(element_type, (config.num_rows, n), divisibility=1)
kernel = CuteSkinnyGemm(
element_type=element_type,
num_rows=config.num_rows,
block_size=config.block_size,
outputs_per_block=config.outputs_per_block,
vector_width=config.vector_width,
k_unroll=config.k_unroll,
has_residual=has_residual,
use_pdl=self._use_pdl(),
)
self._compiled[(dtype, config, has_residual)] = cute.compile(
kernel,
a,
b,
residual,
c,
self._stream(),
options="--enable-tvm-ffi --ptxas-options -maxrregcount=64",
)
def request_warmup(
self,
dtype: torch.dtype,
shapes: Iterable[tuple[int, int, int]],
) -> None:
"""Request compilation of ``(M, N, K)`` shapes before graph capture."""
self.request_warmup_configs(
dtype,
(self._config(m, n, k) for m, n, k in shapes),
)
def request_warmup_configs(
self,
dtype: torch.dtype,
configs: Iterable[SkinnyGemmConfig],
*,
has_residual: bool = False,
) -> None:
"""Request compilation of explicit measured configs before capture."""
self._warmup_configs.update((dtype, config, has_residual) for config in configs)
if self._warmup_registered:
return
from vllm.model_executor.warmup.cutedsl_warmup import (
register_cutedsl_warmup_provider,
)
register_cutedsl_warmup_provider(self)
self._warmup_registered = True
def get_cutedsl_warmup_compile_units(self):
from vllm.model_executor.warmup.cutedsl_warmup import CuTeDSLCompileUnit
return tuple(
CuTeDSLCompileUnit(
name=(
"shape-dynamic skinny GEMM with residual"
if has_residual
else "shape-dynamic skinny GEMM"
),
key=("shape-dynamic-skinny-gemm", dtype, config, has_residual),
compile=partial(self._compile, dtype, config, has_residual),
)
for dtype, config, has_residual in sorted(
self._warmup_configs,
key=lambda item: (
str(item[0]),
item[1].num_rows,
item[1].block_size,
item[1].outputs_per_block,
item[1].k_unroll,
item[1].vector_width,
item[2],
),
)
)
def __call__(
self,
a: torch.Tensor,
b: torch.Tensor,
config: SkinnyGemmConfig | None = None,
residual: torch.Tensor | None = None,
) -> torch.Tensor:
if a.dim() != 2 or b.dim() != 2:
raise ValueError("a and b must be 2D tensors")
if a.dtype not in (torch.bfloat16, torch.float16) or b.dtype != a.dtype:
raise ValueError("a and b must have the same BF16 or FP16 dtype")
if not a.is_cuda or not b.is_cuda or a.device != b.device:
raise ValueError("a and b must be CUDA tensors on the same device")
if not a.is_contiguous() or not b.is_contiguous():
raise ValueError("a and b must be contiguous")
if a.shape[1] != b.shape[1]:
raise ValueError("a and b must have matching K dimensions")
if not 1 <= a.shape[0] <= 16:
raise ValueError("shape-dynamic skinny GEMM requires 1 <= M <= 16")
if residual is not None:
if residual.dim() != 2 or residual.shape != (a.shape[0], b.shape[0]):
raise ValueError("residual must have shape (M, N)")
if residual.dtype != a.dtype:
raise ValueError("residual must have the same dtype as a and b")
if residual.device != a.device or not residual.is_cuda:
raise ValueError("residual must be on the same CUDA device")
if not residual.is_contiguous():
raise ValueError("residual must be contiguous")
config = config or self._config(a.shape[0], b.shape[0], a.shape[1])
if config.num_rows != a.shape[0]:
raise ValueError("config num_rows must match M")
if b.shape[0] % config.outputs_per_block != 0:
raise ValueError("N must be divisible by outputs_per_block")
if a.shape[1] % (config.block_size * config.vector_width) != 0:
raise ValueError(
"K must be divisible by block_size * vector_width for this config"
)
has_residual = residual is not None
cache_key = (a.dtype, config, has_residual)
if cache_key not in self._compiled:
self._compile(a.dtype, config, has_residual)
output = torch.empty((a.shape[0], b.shape[0]), dtype=a.dtype, device=a.device)
residual_arg = output if residual is None else residual
self._compiled[cache_key](a, b, residual_arg, output, self._stream())
return output
shape_dynamic_skinny_gemm = ShapeDynamicSkinnyGemm()
+45
View File
@@ -158,6 +158,51 @@ class SiluAndMul(CustomOp):
return self.forward_native(x)
@CustomOp.register("situ_and_mul")
class SituAndMul(CustomOp):
"""SituGLU activation used by Kimi models.
Computes beta * tanh(gate / beta) * sigmoid(gate) * up. When
``linear_beta`` is set, the up projection is also softly clipped with
linear_beta * tanh(up / linear_beta).
"""
def __init__(
self,
beta: float = 1.0,
linear_beta: float | None = None,
*,
compile_native: bool = True,
):
super().__init__(compile_native=compile_native)
self.beta = float(beta)
self.linear_beta = None if linear_beta is None else float(linear_beta)
if current_platform.is_cuda_alike():
self.op = torch.ops._C.situ_and_mul
def forward_native(self, x: torch.Tensor) -> torch.Tensor:
d = x.shape[-1] // 2
gate = x[..., :d].float()
up = x[..., d:].float()
gate = self.beta * torch.tanh(gate / self.beta) * torch.sigmoid(gate)
if self.linear_beta is not None:
up = self.linear_beta * torch.tanh(up / self.linear_beta)
return (gate * up).to(x.dtype)
def forward_cuda(self, x: torch.Tensor) -> torch.Tensor:
# Fused CUDA kernel: writes straight to `out`, no fp32 temporaries.
# linear_beta<=0 signals "unset" to the kernel (up passed through).
d = x.shape[-1] // 2
out = torch.empty(x.shape[:-1] + (d,), dtype=x.dtype, device=x.device)
self.op(
out, x, self.beta, -1.0 if self.linear_beta is None else self.linear_beta
)
return out
def forward_xpu(self, x: torch.Tensor) -> torch.Tensor:
return self.forward_native(x)
@CustomOp.register("silu_and_mul_with_clamp")
class SiluAndMulWithClamp(CustomOp):
"""SwiGLU activation with input clamping (used by some MoE shared experts).
@@ -22,6 +22,7 @@ class MoEActivation(Enum):
# expects the *packed* layout ([all gates; all ups]), as produced by a
# MergedColumnParallelLinear gate_up_proj (e.g. MiniMax-M3).
SWIGLUOAI = "swigluoai"
SITU = "situ"
SWIGLUOAI_UNINTERLEAVE = "swigluoai_uninterleave"
SWIGLUSTEP = "swiglustep"
@@ -77,6 +78,7 @@ _CUSTOM_OP_NAMES: dict[MoEActivation, str] = {
MoEActivation.SILU: "silu_and_mul",
MoEActivation.GELU: "gelu_and_mul",
MoEActivation.GELU_TANH: "gelu_tanh_and_mul",
MoEActivation.SITU: "situ_and_mul",
MoEActivation.SWIGLUOAI: "swigluoai_and_mul",
MoEActivation.SWIGLUOAI_UNINTERLEAVE: "silu_and_mul_with_clamp",
MoEActivation.SWIGLUSTEP: "swiglustep_and_mul",
@@ -133,6 +135,8 @@ def apply_moe_activation(
beta: float = 0.0,
topk_ids: torch.Tensor | None = None,
expert_map: torch.Tensor | None = None,
activation_situ_beta: float | None = None,
activation_situ_linear_beta: float | None = None,
) -> torch.Tensor:
"""Apply MoE activation function.
@@ -163,6 +167,25 @@ def apply_moe_activation(
torch.ops._C.gelu_and_mul(output, input)
elif activation == MoEActivation.GELU_TANH:
torch.ops._C.gelu_tanh_and_mul(output, input)
elif activation == MoEActivation.SITU:
# Fused CUDA kernel: writes straight to `output`, no fp32 temporaries.
# (The pure-torch fallback below upcast both halves to fp32 and
# allocated ~8 temporaries per call, blowing up MoE memory.)
# Both betas come from FusedMoEConfig; a missing beta means the caller
# bypassed the config plumbing, so fail rather than silently use 1.0.
# linear_beta is genuinely optional: <= 0 signals "unset" to the kernel
# (up passed through), matching SituAndMul(linear_beta=None).
assert activation_situ_beta is not None, (
"SITU requires activation_situ_beta from FusedMoEConfig"
)
torch.ops._C.situ_and_mul(
output,
input,
activation_situ_beta,
-1.0
if activation_situ_linear_beta is None
else activation_situ_linear_beta,
)
elif activation == MoEActivation.SWIGLUOAI:
torch.ops._C.swigluoai_and_mul(output, input)
elif activation == MoEActivation.SWIGLUOAI_UNINTERLEAVE:
@@ -1314,6 +1314,10 @@ class FusedMoEConfig:
swiglu_alpha: float | None = None
swiglu_beta: float | None = None
# SituGLU parameters used by Kimi sit(u/v2) activations.
activation_situ_beta: float | None = None
activation_situ_linear_beta: float | None = None
max_capture_size: int = 0
# Set by __post_init__
@@ -444,7 +444,14 @@ class DeepGemmFP4Experts(mk.FusedMoEExpertsModular):
@staticmethod
def _supports_activation(activation: MoEActivation) -> bool:
return activation in [MoEActivation.SILU, MoEActivation.SWIGLUSTEP]
# SILU has fused gate+mul+quant kernels; SWIGLUSTEP/SITU take the
# general path (activation applied via self.activation, which forwards
# the situ betas, then FP8 requant).
return activation in [
MoEActivation.SILU,
MoEActivation.SWIGLUSTEP,
MoEActivation.SITU,
]
@staticmethod
def _supports_parallel_config(moe_parallel_config: FusedMoEParallelConfig) -> bool:
@@ -488,16 +495,15 @@ class DeepGemmFP4Experts(mk.FusedMoEExpertsModular):
M_sum, N = input.size()
activation_out_dim = self.adjust_N_for_activation(N, activation)
if scale_fmt == DeepGemmQuantScaleFMT.UE8M0:
assert activation == MoEActivation.SILU
return fused_silu_mul_fp8_quant_packed(
input=input,
output_q=output,
group_size=block_k,
clamp_limit=self.gemm1_clamp_limit,
)
if activation == MoEActivation.SILU:
# Fused gate+mul+quant kernels for the common SILU case.
if scale_fmt == DeepGemmQuantScaleFMT.UE8M0:
return fused_silu_mul_fp8_quant_packed(
input=input,
output_q=output,
group_size=block_k,
clamp_limit=self.gemm1_clamp_limit,
)
use_ue8m0 = scale_fmt == DeepGemmQuantScaleFMT.FLOAT32_CEIL_UE8M0
return silu_mul_per_token_group_quant_fp8_colmajor(
input=input,
@@ -506,12 +512,24 @@ class DeepGemmFP4Experts(mk.FusedMoEExpertsModular):
clamp_limit=self.gemm1_clamp_limit,
)
# General gated activations (SWIGLUSTEP, SITU): apply the activation
# (self.activation forwards the situ betas from moe_config) then
# FP8-requant into the layout DeepGEMM expects for this scale format.
act_out = torch.empty(
(M_sum, activation_out_dim), dtype=input.dtype, device=input.device
)
self.activation(activation, act_out, input)
if scale_fmt == DeepGemmQuantScaleFMT.UE8M0:
return per_token_group_quant_fp8_packed_for_deepgemm(
act_out, block_k, use_ue8m0=True, out_q=output
)
use_ue8m0 = scale_fmt == DeepGemmQuantScaleFMT.FLOAT32_CEIL_UE8M0
return per_token_group_quant_fp8(
act_out, block_k, column_major_scales=True, out_q=output
act_out,
block_k,
column_major_scales=True,
out_q=output,
use_ue8m0=use_ue8m0,
)
def apply(
@@ -93,6 +93,8 @@ def _fused_marlin_moe(
clamp_limit: float | None = None,
gemm1_alpha: float = 1.0,
gemm1_beta: float = 0.0,
activation_situ_beta: float | None = None,
activation_situ_linear_beta: float | None = None,
) -> torch.Tensor:
assert hidden_states.ndim == 2
M, K = hidden_states.size()
@@ -171,6 +173,8 @@ def _fused_marlin_moe(
beta=gemm1_beta,
topk_ids=topk_ids,
expert_map=expert_map,
activation_situ_beta=activation_situ_beta,
activation_situ_linear_beta=activation_situ_linear_beta,
)
if output is None:
@@ -256,6 +260,8 @@ def fused_marlin_moe(
clamp_limit: float | None = None,
gemm1_alpha: float = 1.0,
gemm1_beta: float = 0.0,
activation_situ_beta: float | None = None,
activation_situ_linear_beta: float | None = None,
) -> torch.Tensor:
"""
This function computes a Mixture of Experts (MoE) layer using two sets of
@@ -357,6 +363,8 @@ def fused_marlin_moe(
num_tokens_post_padded=num_tokens_post_padded,
activation=activation,
activation_func=activation_func,
activation_situ_beta=activation_situ_beta,
activation_situ_linear_beta=activation_situ_linear_beta,
input_global_scale1=input_global_scale1,
input_global_scale2=input_global_scale2,
global_scale1=global_scale1,
@@ -423,6 +431,9 @@ def batched_fused_marlin_moe(
clamp_limit: float | None = None,
gemm1_alpha: float = 1.0,
gemm1_beta: float = 0.0,
activation_func: Callable[..., None] = apply_moe_activation,
activation_situ_beta: float | None = None,
activation_situ_linear_beta: float | None = None,
) -> torch.Tensor:
"""
This function massages the inputs so the batched hidden_states can be
@@ -530,6 +541,9 @@ def batched_fused_marlin_moe(
quant_type=quant_type,
apply_router_weight_on_input=apply_router_weight_on_input,
activation=activation,
activation_func=activation_func,
activation_situ_beta=activation_situ_beta,
activation_situ_linear_beta=activation_situ_linear_beta,
expert_map=expert_map,
block_size_m=block_size_m,
sorted_token_ids=sorted_token_ids,
@@ -645,6 +659,7 @@ class MarlinExpertsBase(mk.FusedMoEExpertsModular):
MoEActivation.SILU,
MoEActivation.GELU,
MoEActivation.GELU_TANH,
MoEActivation.SITU,
MoEActivation.SWIGLUOAI,
MoEActivation.SWIGLUOAI_UNINTERLEAVE,
MoEActivation.SWIGLUSTEP,
@@ -793,6 +808,10 @@ class MarlinExperts(LoRAExpertsMixin, MarlinExpertsBase):
global_num_experts=global_num_experts,
activation=activation,
activation_func=self.activation,
activation_situ_beta=self.moe_config.activation_situ_beta,
activation_situ_linear_beta=(
self.moe_config.activation_situ_linear_beta
),
moe_sum=self.moe_sum,
expert_map=expert_map,
output=output,
@@ -833,6 +852,8 @@ class MarlinExperts(LoRAExpertsMixin, MarlinExpertsBase):
beta: float = 0.0,
topk_ids: torch.Tensor | None = None,
expert_map: torch.Tensor | None = None,
activation_situ_beta: float | None = None,
activation_situ_linear_beta: float | None = None,
) -> None:
# act_input = intermediate_cache1 (M*topk, 2N for gated)
# act_output = intermediate_cache2 (M*topk, N)
@@ -871,6 +892,8 @@ class MarlinExperts(LoRAExpertsMixin, MarlinExpertsBase):
beta=beta,
topk_ids=topk_ids,
expert_map=expert_map,
activation_situ_beta=activation_situ_beta,
activation_situ_linear_beta=activation_situ_linear_beta,
)
lora_state["cache2"] = act_output
@@ -918,6 +941,8 @@ class MarlinExperts(LoRAExpertsMixin, MarlinExpertsBase):
global_num_experts=global_num_experts,
activation=activation,
activation_func=activation_with_lora,
activation_situ_beta=self.moe_config.activation_situ_beta,
activation_situ_linear_beta=self.moe_config.activation_situ_linear_beta,
moe_sum=moe_sum_with_lora,
expert_map=expert_map,
output=output,
@@ -1021,6 +1046,41 @@ class BatchedMarlinExperts(MarlinExpertsBase):
apply_router_weight_on_input: bool,
):
assert expert_tokens_meta is not None, "Num valid tokens per batch is required"
def activation_func(
act: MoEActivation,
act_output: torch.Tensor,
act_input: torch.Tensor,
*,
activation_situ_beta: float | None = None,
activation_situ_linear_beta: float | None = None,
**kwargs,
) -> None:
if act != MoEActivation.SITU:
self.activation(
act,
act_output,
act_input,
activation_situ_beta=activation_situ_beta,
activation_situ_linear_beta=activation_situ_linear_beta,
**kwargs,
)
return
num_experts, max_num_tokens = hidden_states.shape[:2]
beta = activation_situ_beta
linear_beta = activation_situ_linear_beta
assert beta is not None, (
"SITU requires activation_situ_beta from FusedMoEConfig"
)
torch.ops._C.masked_situ_and_mul(
act_output.view(num_experts, max_num_tokens, -1),
act_input.view(num_experts, max_num_tokens, -1),
expert_tokens_meta.expert_num_tokens,
beta,
-1.0 if linear_beta is None else linear_beta,
)
return batched_fused_marlin_moe(
hidden_states=hidden_states,
expert_num_tokens=expert_tokens_meta.expert_num_tokens,
@@ -1051,4 +1111,7 @@ class BatchedMarlinExperts(MarlinExpertsBase):
clamp_limit=self.gemm1_clamp_limit,
gemm1_alpha=self.gemm1_alpha,
gemm1_beta=self.gemm1_beta,
activation_func=activation_func,
activation_situ_beta=self.moe_config.activation_situ_beta,
activation_situ_linear_beta=self.moe_config.activation_situ_linear_beta,
)
@@ -79,6 +79,36 @@ class TrtLlmMxfp4ExpertsBase:
else:
self.gemm1_clamp_limit = None
# SITU (SituGLU) TRTLLM-Gen kernel computes
# left = alpha * tanh(x0 / alpha) * sigmoid(x0) # gate (x0)
# right = beta * tanh(x1 / beta) # up (x1)
# which matches vLLM's situ_and_mul with (beta, linear_beta), so map
# situ beta -> gatedActAlpha (gemm1_alpha) and situ linear_beta ->
# gatedActBeta (gemm1_beta). Both must be > 0.
if moe_config.activation == MoEActivation.SITU:
situ_beta = moe_config.activation_situ_beta
situ_linear_beta = moe_config.activation_situ_linear_beta
assert situ_beta is not None and situ_beta > 0, (
"SITU requires activation_situ_beta > 0"
)
assert situ_linear_beta is not None and situ_linear_beta > 0, (
"TRTLLM SiTuGlu requires activation_situ_linear_beta > 0 "
"(the private cubin has no up-passthrough path)"
)
self.gemm1_alpha = torch.full(
(self.local_num_experts,),
float(situ_beta),
dtype=torch.float32,
device=device,
)
self.gemm1_beta = torch.full(
(self.local_num_experts,),
float(situ_linear_beta),
dtype=torch.float32,
device=device,
)
self.gemm1_clamp_limit = None
self.max_capture_size = moe_config.max_capture_size
@staticmethod
@@ -103,7 +133,19 @@ class TrtLlmMxfp4ExpertsBase:
@staticmethod
def _supports_activation(activation: MoEActivation) -> bool:
return activation in (MoEActivation.SWIGLUOAI, MoEActivation.SILU)
return activation in (
MoEActivation.SWIGLUOAI,
MoEActivation.SILU,
MoEActivation.SITU,
)
@staticmethod
def _flashinfer_activation_type(activation: MoEActivation) -> int:
from flashinfer.fused_moe.core import ActivationType
if activation == MoEActivation.SITU:
return ActivationType.Situ.value
return ActivationType.Swiglu.value
@staticmethod
def activation_format() -> mk.FusedMoEActivationFormat:
@@ -142,6 +184,7 @@ class TrtLlmMxfp4ExpertsMonolithic(
activation_key: QuantKey | None,
) -> bool:
return routing_method in [
RoutingMethodType.DeepSeekV3,
RoutingMethodType.Renormalize,
RoutingMethodType.RenormalizeNaive,
]
@@ -192,8 +235,8 @@ class TrtLlmMxfp4ExpertsMonolithic(
device=hidden_states.device,
)
trtllm_fp4_block_scale_moe(
routing_logits=router_logits.to(torch.bfloat16),
routing_bias=None,
routing_logits=router_logits,
routing_bias=e_score_correction_bias,
hidden_states=x_quant,
hidden_states_scale=x_scale,
gemm1_weights=w1,
@@ -210,14 +253,15 @@ class TrtLlmMxfp4ExpertsMonolithic(
output2_scale_scalar=None,
num_experts=global_num_experts,
top_k=self.topk,
n_group=None,
topk_group=None,
n_group=(num_expert_group or 0),
topk_group=(topk_group or 0),
intermediate_size=self.intermediate_size_per_partition,
local_expert_offset=self.ep_rank * self.local_num_experts,
local_num_experts=self.local_num_experts,
routed_scaling_factor=None,
routed_scaling_factor=routed_scaling_factor,
routing_method_type=self.routing_method_type,
do_finalize=True,
activation_type=self._flashinfer_activation_type(activation),
tune_max_num_tokens=max(self.max_capture_size, 1),
output=output,
routing_replay_out=routing_replay_out,
@@ -338,6 +382,7 @@ class TrtLlmMxfp4ExpertsModular(TrtLlmMxfp4ExpertsBase, mk.FusedMoEExpertsModula
"routing_method_type": RoutingMethodType.Renormalize,
"do_finalize": True,
"enable_pdl": True,
"activation_type": self._flashinfer_activation_type(activation),
"output": output,
"tune_max_num_tokens": max(self.max_capture_size, 1),
}
@@ -110,6 +110,27 @@ class TrtLlmNvFp4ExpertsBase:
self.gemm1_alpha = None
self.gemm1_beta = None
# SITU (Kimi SituGLU) TRTLLM-Gen kernel computes
# left=alpha*tanh(x0/alpha)*sigmoid(x0), right=beta*tanh(x1/beta),
# matching vLLM's situ_and_mul, so map situ beta -> gatedActAlpha
# (gemm1_alpha) and situ linear_beta -> gatedActBeta (gemm1_beta).
# These operate on the dequantized gate/up, so they are NOT folded by
# g1_alphas in process_weights_after_loading.
self.is_situ = moe_config.activation == MoEActivation.SITU
if self.is_situ:
situ_beta = moe_config.activation_situ_beta
situ_linear_beta = moe_config.activation_situ_linear_beta
assert situ_beta is not None and situ_beta > 0, (
"SITU requires activation_situ_beta > 0"
)
assert situ_linear_beta is not None and situ_linear_beta > 0, (
"TRTLLM SiTuGlu requires activation_situ_linear_beta > 0 "
"(the private cubin has no up-passthrough path)"
)
self.gemm1_alpha = _per_expert(situ_beta)
self.gemm1_beta = _per_expert(situ_linear_beta)
self.gemm1_clamp_limit = None
logger.debug_once(
"activation=%s, gemm1_alpha=%s, gemm1_beta=%s, gemm1_clamp_limit=%s",
moe_config.activation,
@@ -142,7 +163,10 @@ class TrtLlmNvFp4ExpertsBase:
# (via the in-place mul above) and never changes again, so this is a
# static, per-expert constant. Register on the layer so EPLB
# rearranges it alongside the other expert tensors.
if self.gemm1_clamp_limit is not None:
# SITU alpha/beta act on the dequantized gate/up (tanh clamps), not the
# raw GEMM1 accumulator, so they are registered as-is without the
# g1_alphas fold used by the SwiGLU-OAI clamp/beta below.
if self.gemm1_clamp_limit is not None and not self.is_situ:
gemm1_clamp_limit = self.gemm1_clamp_limit / self.quant_config.g1_alphas
layer.register_parameter(
"gemm1_clamp_limit",
@@ -155,7 +179,11 @@ class TrtLlmNvFp4ExpertsBase:
# raw. Register both on the layer so EPLB rearranges them with the
# other per-expert tensors.
if self.gemm1_beta is not None:
gemm1_beta = self.gemm1_beta / self.quant_config.g1_alphas
gemm1_beta = (
self.gemm1_beta
if self.is_situ
else self.gemm1_beta / self.quant_config.g1_alphas
)
layer.register_parameter(
"gemm1_beta",
torch.nn.Parameter(gemm1_beta, requires_grad=False),
@@ -197,13 +225,15 @@ class TrtLlmNvFp4ExpertsBase:
@staticmethod
def _supports_activation(activation: MoEActivation) -> bool:
"""Supports SiLU, RELU^2 non-gated, GELU, and clamped SwiGLU-OAI."""
"""Supports SiLU, RELU^2 non-gated, GELU, clamped SwiGLU-OAI, and the
private SituGLU (SITU) cubin."""
return activation in [
MoEActivation.SILU,
MoEActivation.RELU2_NO_MUL,
MoEActivation.GELU,
MoEActivation.GELU_TANH,
MoEActivation.SWIGLUOAI_UNINTERLEAVE,
MoEActivation.SITU,
]
@staticmethod
@@ -120,6 +120,8 @@ def FusedMoE(
swiglu_limit: float | None = None,
swiglu_alpha: float | None = None,
swiglu_beta: float | None = None,
activation_situ_beta: float | None = None,
activation_situ_linear_beta: float | None = None,
e_score_correction_bias: torch.Tensor | None = None,
apply_router_weight_on_input: bool = False,
activation: str = "silu",
@@ -178,6 +180,8 @@ def FusedMoE(
scoring_func: Scoring function for routing ("softmax" or others)
routed_scaling_factor: Scaling factor applied to topk_weights or output
swiglu_limit: SwiGLU activation limit
activation_situ_beta: SituGLU activation beta
activation_situ_linear_beta: SituGLU linear beta
e_score_correction_bias: Expert score correction bias tensor
apply_router_weight_on_input: Whether to apply router weights on input
activation: Activation function name ("silu", "gelu", etc.)
@@ -352,6 +356,8 @@ def FusedMoE(
swiglu_limit=swiglu_limit,
swiglu_alpha=swiglu_alpha,
swiglu_beta=swiglu_beta,
activation_situ_beta=activation_situ_beta,
activation_situ_linear_beta=activation_situ_linear_beta,
max_capture_size=vllm_config.compilation_config.max_cudagraph_capture_size,
skip_final_all_reduce=skip_final_all_reduce,
)
@@ -890,6 +890,8 @@ class FusedMoEExpertsModular(FusedMoEExperts):
beta: float = 0.0,
topk_ids: torch.Tensor | None = None,
expert_map: torch.Tensor | None = None,
activation_situ_beta: float | None = None,
activation_situ_linear_beta: float | None = None,
) -> None:
apply_moe_activation(
activation,
@@ -900,6 +902,16 @@ class FusedMoEExpertsModular(FusedMoEExperts):
beta=beta,
topk_ids=topk_ids,
expert_map=expert_map,
activation_situ_beta=(
self.moe_config.activation_situ_beta
if activation_situ_beta is None
else activation_situ_beta
),
activation_situ_linear_beta=(
self.moe_config.activation_situ_linear_beta
if activation_situ_linear_beta is None
else activation_situ_linear_beta
),
)
@abstractmethod
@@ -1307,6 +1319,14 @@ class FusedMoEKernelModularImpl:
activation,
)
use_output_alias = (
output_alias is not None
and output_alias.shape == fused_out.shape
and output_alias.dtype == fused_out.dtype
and output_alias.device == fused_out.device
and output_alias.is_contiguous()
)
# If caller's output buffer already matches fused_out shape/dtype, alias
# to skip the redundant copy in TopKWeightAndReduceNoOP.apply downstream.
# This eliminates ~94% of __amd_rocclr_copyBuffer events (Copy 2 of the
@@ -1314,15 +1334,10 @@ class FusedMoEKernelModularImpl:
if current_platform.is_rocm():
from vllm._aiter_ops import rocm_aiter_ops
if (
rocm_aiter_ops.is_fused_moe_enabled()
and output_alias is not None
and output_alias.shape == fused_out.shape
and output_alias.dtype == fused_out.dtype
and output_alias.device == fused_out.device
and output_alias.is_contiguous()
):
if use_output_alias and rocm_aiter_ops.is_fused_moe_enabled():
fused_out = output_alias
elif use_output_alias:
fused_out = output_alias
self.fused_experts.apply(
output=fused_out,
@@ -0,0 +1,255 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
import vllm.envs as envs
from vllm.config import get_current_vllm_config
from vllm.distributed import tensor_model_parallel_all_reduce
from vllm.logger import init_logger
from vllm.model_executor.layers.fused_allreduce_gemma_rms_norm import (
_AR_RESIDUAL_RMS_NORM,
_can_use_flashinfer,
flashinfer_trtllm_fused_allreduce_norm,
)
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.utils.torch_utils import aux_stream, current_stream
from .moe_runner import MoERunner, _unpack
logger = init_logger(__name__)
class LatentMoERunner(MoERunner):
"""MoE runner for latent MoE with a replicated routed up-projection.
Fused path (tp>1, un-reduced combine output, shared expert, no SP):
concatenates the un-reduced latent partial (dim d) and the un-reduced
shared partial (dim D) into one contiguous buffer, all-reduces once, then
splits. The latent half is normed and up-projected locally (replicated
up-proj -> full hidden), and the shared add folds into the GEMM epilogue
(``torch.addmm``). One collective total, no post-reduction communication.
Native path: the replicated up-proj produces the full hidden dim on every
rank, so the base runner combines routed + shared correctly at any TP size
(using two collectives instead of the fused path's one).
"""
def __init__(
self,
*args,
enable_k3_latent_moe_tail_fusion: bool = False,
**kwargs,
) -> None:
super().__init__(*args, **kwargs)
self.enable_k3_latent_moe_tail_fusion = enable_k3_latent_moe_tail_fusion
use_fused_path = self._use_fused_path()
if (
self.enable_k3_latent_moe_tail_fusion
and use_fused_path
and self.moe_config.tp_size not in (8, 16)
):
logger.warning_once(
"K3 latent-MoE tail fusion currently supports TP=8 and TP=16, "
"but TP=%d is configured. Falling back to the default path.",
self.moe_config.tp_size,
)
self.enable_k3_latent_moe_tail_fusion = False
if self.enable_k3_latent_moe_tail_fusion and use_fused_path:
vllm_config = get_current_vllm_config()
if vllm_config.parallel_config.use_ubatching:
raise ValueError(
"K3 latent-MoE tail fusion does not support DBO or ubatching."
)
if vllm_config.model_config.enable_sleep_mode:
raise ValueError(
"K3 latent-MoE tail fusion does not support sleep mode."
)
transform = self.routed_output_transform
assert transform is not None
norm = transform.norm
assert norm is not None
from vllm.models.kimi_k3.nvidia.ops.latent_moe_tail import (
KimiK3LatentMoETailOp,
)
op = KimiK3LatentMoETailOp.initialize(
hidden_size=transform.up_proj.weight.shape[0],
latent_size=norm.weight.shape[0],
dtype=norm.weight.dtype,
device=norm.weight.device,
rms_eps=norm.variance_epsilon,
)
self._k3_latent_moe_tail_op = op
def _get_zero_residual(self, hidden_states: torch.Tensor) -> torch.Tensor:
"""Read-only zero ``residual_in`` for the fused AR+RMSNorm kernel.
flashinfer requires a residual buffer even when there is no residual to
add.
"""
numel = hidden_states.numel()
buf = getattr(self, "_zero_residual", None)
if (
buf is None
or buf.dtype != hidden_states.dtype
or buf.device != hidden_states.device
or buf.numel() < numel
):
buf = torch.zeros(
numel, dtype=hidden_states.dtype, device=hidden_states.device
)
self._zero_residual = buf
return buf[:numel].view_as(hidden_states)
def _use_fused_path(self) -> bool:
# The fused path merges the latent and shared reductions into one
# all-reduce, so it needs actual TP parallelism, a shared expert (to
# concat), an un-reduced combine output, and no sequence parallelism.
return (
self.moe_config.tp_size > 1
and self._shared_experts is not None
and not self._fused_output_is_reduced
and not self.moe_config.is_sequence_parallel
)
def forward(
self,
hidden_states: torch.Tensor,
router_logits: torch.Tensor,
input_ids: torch.Tensor | None = None,
shared_experts_input: torch.Tensor | None = None,
) -> torch.Tensor:
if self._use_fused_path():
return self._fused_forward(
hidden_states, router_logits, input_ids, shared_experts_input
)
return super().forward(
hidden_states, router_logits, input_ids, shared_experts_input
)
def _fused_forward(
self,
hidden_states: torch.Tensor,
router_logits: torch.Tensor,
input_ids: torch.Tensor | None = None,
shared_experts_input: torch.Tensor | None = None,
) -> torch.Tensor:
# When the caller pre-applies the routed input transform outside the
# runner (e.g. to overlap it on a separate stream), it passes the
# already-transformed routed input as ``hidden_states`` and the original
# hidden states as ``shared_experts_input``; skip the transform then.
if shared_experts_input is None:
hidden_states, shared_experts_input = self.apply_routed_input_transform(
hidden_states
)
hidden_states, og_hidden_dim_pre_xform, og_hidden_dim_post_xform = (
self._maybe_pad_hidden_states(
shared_experts_input,
hidden_states,
)
)
result = self._forward_entry(
hidden_states,
router_logits,
shared_experts_input,
input_ids,
self._encode_layer_name(),
self.moe_config.hidden_dim_unpadded
if self._quant_method.has_unpadded_output
else 0,
)
shared_output, fused_output = _unpack(result)
assert shared_output is not None
if og_hidden_dim_pre_xform is not None:
fused_output = fused_output[..., :og_hidden_dim_pre_xform]
transform = self.routed_output_transform
assert transform is not None
if self.enable_k3_latent_moe_tail_fusion:
op = self._k3_latent_moe_tail_op
if 0 < fused_output.shape[0] <= op.contract.max_num_tokens:
norm = transform.norm
assert norm is not None
result = op(
fused_output,
shared_output,
norm.weight,
transform.up_proj.weight,
)
result = self._maybe_reduce_final_output(
result, og_hidden_dim_post_xform, output_is_reduced=True
)
return self._maybe_add_zero_expert_output(result)
fused_latent = None
if transform.norm is not None:
fused_latent = self.allreduce_norm_latent_out(fused_output, transform.norm)
else:
fused_latent = tensor_model_parallel_all_reduce(fused_output)
shared_expert_stream = (
aux_stream()
if shared_output.size(0) <= envs.VLLM_SHARED_EXPERTS_STREAM_TOKEN_THRESHOLD
else None
)
if shared_expert_stream is not None:
# overlap shared expert allreduce with latent up_proj
main = current_stream()
shared_output.record_stream(shared_expert_stream)
shared_expert_stream.wait_stream(main)
with torch.cuda.stream(shared_expert_stream):
shared_output = tensor_model_parallel_all_reduce(shared_output)
result = torch.mm(fused_latent, transform.up_proj.weight.t())
main.wait_stream(shared_expert_stream)
else:
shared_output = tensor_model_parallel_all_reduce(shared_output)
result = torch.mm(fused_latent, transform.up_proj.weight.t())
result.add_(shared_output)
# Output is already fully reduced; this only strips padding.
result = self._maybe_reduce_final_output(
result, og_hidden_dim_post_xform, output_is_reduced=True
)
return self._maybe_add_zero_expert_output(result)
def allreduce_norm_latent_out(
self,
hidden_states: torch.Tensor,
norm: RMSNorm,
) -> tuple[torch.Tensor, torch.Tensor]:
"""All-reduce + add residual + (standard) RMSNorm, fused via flashinfer."""
if self.moe_config.tp_size == 1:
return norm(hidden_states)
if flashinfer_trtllm_fused_allreduce_norm is not None:
ok, max_token_num = _can_use_flashinfer(
hidden_states, self.moe_config.tp_size
)
if ok:
norm_out = torch.empty_like(hidden_states)
# With norm_out provided, the kernel writes the new residual
# (all_reduce(hidden_states) + residual) into the hidden_states
# buffer and the normalized result into norm_out.
flashinfer_trtllm_fused_allreduce_norm(
allreduce_in=hidden_states,
residual=self._get_zero_residual(hidden_states),
rms_gamma=norm.weight,
rms_eps=norm.variance_epsilon,
world_size=self.moe_config.tp_size,
weight_bias=0.0,
launch_with_pdl=True,
fp32_acc=True,
max_token_num=max_token_num,
pattern_code=_AR_RESIDUAL_RMS_NORM,
norm_out=norm_out,
)
return norm_out
reduced = tensor_model_parallel_all_reduce(hidden_states)
return norm(reduced)
@@ -348,7 +348,8 @@ class MoERunner(MoERunnerInterface):
return self.routed_experts.quant_method
def apply_routed_input_transform(
self, hidden_states: torch.Tensor
self,
hidden_states: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor | None]:
"""Apply transform for routed experts (e.g., latent projection).
@@ -416,6 +417,7 @@ class MoERunner(MoERunnerInterface):
def _maybe_reduce_shared_expert_output(
self,
shared_output: torch.Tensor | None,
fused_output_is_reduced: bool | None = None,
) -> torch.Tensor | None:
"""All-reduce shared expert output when the combine kernel already
reduced fused output.
@@ -425,18 +427,43 @@ class MoERunner(MoERunnerInterface):
* If we have SP (TP=N, DP=M, EP), there is a separate AG step handled
in the model.
"""
if fused_output_is_reduced is None:
fused_output_is_reduced = self._fused_output_is_reduced
if (
shared_output is not None
and not self.moe_config.is_sequence_parallel
and self._fused_output_is_reduced
and fused_output_is_reduced
):
shared_output = tensor_model_parallel_all_reduce(shared_output)
return shared_output
def _maybe_reduce_routed_output_before_transform(
self,
fused_output: torch.Tensor,
fused_output_is_reduced: bool,
) -> tuple[torch.Tensor, bool]:
"""All-reduce latent routed output before its output transform.
Latent MoE output transforms may contain non-linear ops, e.g. RMSNorm.
TP partial routed outputs must be summed in latent space before such
transforms are applied.
"""
if (
self.routed_output_transform is not None
and not self.moe_config.is_sequence_parallel
and (self.moe_config.tp_size > 1 or self.moe_config.ep_size > 1)
and not fused_output_is_reduced
):
fused_output = tensor_model_parallel_all_reduce(fused_output)
fused_output_is_reduced = True
return fused_output, fused_output_is_reduced
def _maybe_reduce_final_output(
self,
states: torch.Tensor,
trunc_size: int | None,
output_is_reduced: bool | None = None,
) -> torch.Tensor:
"""All-reduce the combined output if needed.
@@ -455,11 +482,14 @@ class MoERunner(MoERunnerInterface):
# We don't need to reduce the final output if:
# - We are not running with TP or DP
# - The MK already reduced the fused output itself.
if output_is_reduced is None:
output_is_reduced = self._fused_output_is_reduced
if (
not self.moe_config.is_sequence_parallel
and not self.moe_config.skip_final_all_reduce
and (self.moe_config.tp_size > 1 or self.moe_config.ep_size > 1)
and not self._fused_output_is_reduced
and not output_is_reduced
):
states = tensor_model_parallel_all_reduce(states)
@@ -643,6 +673,7 @@ class MoERunner(MoERunnerInterface):
hidden_states: torch.Tensor,
router_logits: torch.Tensor,
input_ids: torch.Tensor | None = None,
shared_experts_input: torch.Tensor | None = None,
) -> torch.Tensor:
"""Invoke the fused moe layer.
@@ -664,11 +695,16 @@ class MoERunner(MoERunnerInterface):
_moe_forward and _moe_forward_shared must be split.
"""
# Apply transform for routed experts (e.g., latent projection
# for latent MoE)
hidden_states, shared_experts_input = self.apply_routed_input_transform(
hidden_states
)
# Apply transform for routed experts (e.g., latent projection for
# latent MoE). When the caller pre-applies the routed input transform
# outside the runner (e.g. to overlap it on a separate stream), it
# passes the already-transformed routed input as ``hidden_states`` and
# the original hidden states as ``shared_experts_input``; skip the
# transform in that case so shared experts still see the original input.
if shared_experts_input is None:
hidden_states, shared_experts_input = self.apply_routed_input_transform(
hidden_states
)
# Record before `_maybe_pad_hidden_states` pads activations to match
# `moe_config.hidden_dim`, e.g. after `align_trtllm_fp4_moe_hidden_dim_for_fi`
@@ -708,9 +744,22 @@ class MoERunner(MoERunnerInterface):
if og_hidden_dim_pre_xform is not None:
fused_output = fused_output[..., :og_hidden_dim_pre_xform]
# If combine kernel already reduced fused, reduce shared to match.
fused_output_is_reduced = self._fused_output_is_reduced
# Latent routed output has to be reduced before output transform,
# because the transform may include non-linear normalization.
fused_output, fused_output_is_reduced = (
self._maybe_reduce_routed_output_before_transform(
fused_output,
fused_output_is_reduced,
)
)
# If routed output is already reduced, reduce shared to match.
# See note above re: the two all-reduce points.
shared_output = self._maybe_reduce_shared_expert_output(shared_output)
shared_output = self._maybe_reduce_shared_expert_output(
shared_output, fused_output_is_reduced
)
shared_output, fused_output = self._maybe_apply_routed_scale_to_output(
shared_output, fused_output
@@ -724,7 +773,9 @@ class MoERunner(MoERunnerInterface):
else:
result = fused_output
result = self._maybe_reduce_final_output(result, og_hidden_dim_post_xform)
result = self._maybe_reduce_final_output(
result, og_hidden_dim_post_xform, fused_output_is_reduced
)
return self._maybe_add_zero_expert_output(result)
@@ -36,6 +36,7 @@ class MoERunnerInterface(PluggableLayer, ABC):
hidden_states: torch.Tensor,
router_logits: torch.Tensor,
input_ids: torch.Tensor | None = None,
shared_experts_input: torch.Tensor | None = None,
) -> torch.Tensor:
raise NotImplementedError
@@ -1080,6 +1080,7 @@ def causal_conv1d_update(
block_idx_last_scheduled_token: torch.Tensor | None = None,
initial_state_idx: torch.Tensor | None = None,
validate_data=False,
out: torch.Tensor | None = None,
):
"""
x: Input tensor which can take the following shapes:
@@ -1117,7 +1118,8 @@ def causal_conv1d_update(
for example: conv_state_indices = [null_block_id, 1, 20, null_block_id]
in this case, the kernel will not process entries at
indices 0 and 3
out: (batch, dim) or (batch, dim, seqlen) or (num_tokens, dim), same shape as `x`
out: optional output tensor with the same shape as `x`. When omitted,
the input is overwritten.
"""
if validate_data:
assert null_block_id is not None
@@ -1129,10 +1131,22 @@ def causal_conv1d_update(
original_x_dtype = x.dtype
x = x.to(conv_state.dtype)
if out is None:
out = x
else:
if out.shape != x.shape:
raise ValueError(
f"`out` shape {tuple(out.shape)} must match `x` shape {tuple(x.shape)}."
)
if out.dtype != original_x_dtype or out.device != x.device:
raise ValueError(
"`out` must have the same dtype and device as the input `x`."
)
unsqueeze = query_start_loc is None and x.dim() == 2
if unsqueeze:
# make it (batch, dim, seqlen) with seqlen == 1
x = x.unsqueeze(-1)
out = out.unsqueeze(-1)
if query_start_loc is None:
batch, dim, seqlen = x.shape
else:
@@ -1159,8 +1173,6 @@ def causal_conv1d_update(
assert num_cache_lines >= batch
assert weight.stride(1) == 1 # Need this
# adopt the strategy in vLLM that overwrite on 'x' directly, rather than creating a new tensor 'o'
out = x
stride_w_dim, stride_w_width = weight.stride()
if query_start_loc is None:
@@ -0,0 +1,83 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
from vllm.platforms import current_platform
from vllm.triton_utils import tl, triton
@triton.jit
def _gather_initial_states_kernel(
state_ptr,
indices_ptr,
has_initial_state_ptr,
output_ptr,
stride_state_batch,
stride_indices,
stride_has_initial_state,
row_size: tl.constexpr,
BLOCK_SIZE: tl.constexpr,
launch_pdl: tl.constexpr,
):
block_idx = tl.program_id(0)
batch_idx = tl.program_id(1)
offsets = block_idx * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)
mask = offsets < row_size
if launch_pdl:
tl.extra.cuda.gdc_wait()
tl.extra.cuda.gdc_launch_dependents()
has_initial_state = tl.load(
has_initial_state_ptr + batch_idx * stride_has_initial_state
).to(tl.int1)
state_idx = tl.load(indices_ptr + batch_idx * stride_indices).to(tl.int64)
state_idx = tl.where(has_initial_state, state_idx, 0)
values = tl.load(
state_ptr + state_idx * stride_state_batch + offsets,
mask=mask & has_initial_state,
other=0.0,
)
tl.store(output_ptr + batch_idx * row_size + offsets, values, mask=mask)
def gather_initial_states(
state: torch.Tensor,
indices: torch.Tensor,
has_initial_state: torch.Tensor,
) -> torch.Tensor:
"""Gather dense state rows, replacing uninitialized rows with zeros."""
assert state.ndim >= 2
assert state.is_cuda
assert indices.ndim == 1 and has_initial_state.ndim == 1
assert indices.shape == has_initial_state.shape
assert indices.device == state.device
assert has_initial_state.device == state.device
assert indices.dtype in (torch.int32, torch.int64)
assert has_initial_state.dtype == torch.bool
row_size = state[0].numel()
# Mamba pages may pad stride(0), but each state row remains dense.
assert state[0].is_contiguous()
output = torch.empty(
(indices.numel(), *state.shape[1:]),
dtype=state.dtype,
device=state.device,
)
block_size = min(triton.next_power_of_2(row_size), 1024)
grid = (triton.cdiv(row_size, block_size), indices.numel())
_gather_initial_states_kernel[grid](
state,
indices,
has_initial_state,
output,
state.stride(0),
indices.stride(0),
has_initial_state.stride(0),
row_size=row_size,
BLOCK_SIZE=block_size,
num_warps=8,
launch_pdl=current_platform.is_arch_support_pdl(),
)
return output