[Model] Add Kimi K3 support: model files and kernels [1/N] (#50089)
Signed-off-by: zjy0516 <riverclouds.zhu@qq.com> Signed-off-by: Isotr0py <Isotr0py@outlook.com> Signed-off-by: Bugen Zhao <i@bugenzhao.com> Signed-off-by: Yongye Zhu <zyy1102000@gmail.com> Co-authored-by: Isotr0py <Isotr0py@outlook.com> Co-authored-by: Thien Tran <gau.nernst@yahoo.com.sg> Co-authored-by: Bugen Zhao <i@bugenzhao.com> Co-authored-by: Yongye Zhu <zyy1102000@gmail.com> Co-authored-by: Ziming Huang <zelda.huanghuang@gmail.com> Co-authored-by: Roger Wang <hey@rogerw.io> Co-authored-by: Isotr0py <mozf@inferact.ai> Co-authored-by: aoshen02 <aoshen02@users.noreply.github.com> Co-authored-by: Woosuk Kwon <woosuk@inferact.ai> Co-authored-by: Jee Jee Li <jeejeelee@inferact.ai> Co-authored-by: aoshen02 <aoshen@inferact.ai> Co-authored-by: Summer Yang <girasoleyang@gmail.com> Co-authored-by: Kevin H. Luu <khluu000@gmail.com> Co-authored-by: Bowen Wang <abmfy@icloud.com> Co-authored-by: gnovack <novackgm@gmail.com> Co-authored-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai> Co-authored-by: xiaozhoupy <peiyuanzhou1994@gmail.com> Co-authored-by: Roy Wang <yasong.wang@inferact.ai> Co-authored-by: Jeff (Junze) Ma <93145857+majunze2001@users.noreply.github.com> Co-authored-by: Codex <codex@openai.com>
This commit is contained in:
co-authored by
Isotr0py
Thien Tran
Bugen Zhao
Yongye Zhu
Ziming Huang
Roger Wang
Isotr0py
aoshen02
Woosuk Kwon
Jee Jee Li
aoshen02
Summer Yang
Kevin H. Luu
Bowen Wang
gnovack
Nick Hill
Yifan Qiao
xiaozhoupy
Roy Wang
Jeff Ma
Codex
parent
6f00a1ae3b
commit
7c6729b769
@@ -158,6 +158,51 @@ class SiluAndMul(CustomOp):
|
||||
return self.forward_native(x)
|
||||
|
||||
|
||||
@CustomOp.register("situ_and_mul")
|
||||
class SituAndMul(CustomOp):
|
||||
"""SituGLU activation used by Kimi models.
|
||||
|
||||
Computes beta * tanh(gate / beta) * sigmoid(gate) * up. When
|
||||
``linear_beta`` is set, the up projection is also softly clipped with
|
||||
linear_beta * tanh(up / linear_beta).
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
beta: float = 1.0,
|
||||
linear_beta: float | None = None,
|
||||
*,
|
||||
compile_native: bool = True,
|
||||
):
|
||||
super().__init__(compile_native=compile_native)
|
||||
self.beta = float(beta)
|
||||
self.linear_beta = None if linear_beta is None else float(linear_beta)
|
||||
if current_platform.is_cuda_alike():
|
||||
self.op = torch.ops._C.situ_and_mul
|
||||
|
||||
def forward_native(self, x: torch.Tensor) -> torch.Tensor:
|
||||
d = x.shape[-1] // 2
|
||||
gate = x[..., :d].float()
|
||||
up = x[..., d:].float()
|
||||
gate = self.beta * torch.tanh(gate / self.beta) * torch.sigmoid(gate)
|
||||
if self.linear_beta is not None:
|
||||
up = self.linear_beta * torch.tanh(up / self.linear_beta)
|
||||
return (gate * up).to(x.dtype)
|
||||
|
||||
def forward_cuda(self, x: torch.Tensor) -> torch.Tensor:
|
||||
# Fused CUDA kernel: writes straight to `out`, no fp32 temporaries.
|
||||
# linear_beta<=0 signals "unset" to the kernel (up passed through).
|
||||
d = x.shape[-1] // 2
|
||||
out = torch.empty(x.shape[:-1] + (d,), dtype=x.dtype, device=x.device)
|
||||
self.op(
|
||||
out, x, self.beta, -1.0 if self.linear_beta is None else self.linear_beta
|
||||
)
|
||||
return out
|
||||
|
||||
def forward_xpu(self, x: torch.Tensor) -> torch.Tensor:
|
||||
return self.forward_native(x)
|
||||
|
||||
|
||||
@CustomOp.register("silu_and_mul_with_clamp")
|
||||
class SiluAndMulWithClamp(CustomOp):
|
||||
"""SwiGLU activation with input clamping (used by some MoE shared experts).
|
||||
|
||||
@@ -22,6 +22,7 @@ class MoEActivation(Enum):
|
||||
# expects the *packed* layout ([all gates; all ups]), as produced by a
|
||||
# MergedColumnParallelLinear gate_up_proj (e.g. MiniMax-M3).
|
||||
SWIGLUOAI = "swigluoai"
|
||||
SITU = "situ"
|
||||
SWIGLUOAI_UNINTERLEAVE = "swigluoai_uninterleave"
|
||||
SWIGLUSTEP = "swiglustep"
|
||||
|
||||
@@ -77,6 +78,7 @@ _CUSTOM_OP_NAMES: dict[MoEActivation, str] = {
|
||||
MoEActivation.SILU: "silu_and_mul",
|
||||
MoEActivation.GELU: "gelu_and_mul",
|
||||
MoEActivation.GELU_TANH: "gelu_tanh_and_mul",
|
||||
MoEActivation.SITU: "situ_and_mul",
|
||||
MoEActivation.SWIGLUOAI: "swigluoai_and_mul",
|
||||
MoEActivation.SWIGLUOAI_UNINTERLEAVE: "silu_and_mul_with_clamp",
|
||||
MoEActivation.SWIGLUSTEP: "swiglustep_and_mul",
|
||||
@@ -133,6 +135,8 @@ def apply_moe_activation(
|
||||
beta: float = 0.0,
|
||||
topk_ids: torch.Tensor | None = None,
|
||||
expert_map: torch.Tensor | None = None,
|
||||
activation_situ_beta: float | None = None,
|
||||
activation_situ_linear_beta: float | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""Apply MoE activation function.
|
||||
|
||||
@@ -163,6 +167,25 @@ def apply_moe_activation(
|
||||
torch.ops._C.gelu_and_mul(output, input)
|
||||
elif activation == MoEActivation.GELU_TANH:
|
||||
torch.ops._C.gelu_tanh_and_mul(output, input)
|
||||
elif activation == MoEActivation.SITU:
|
||||
# Fused CUDA kernel: writes straight to `output`, no fp32 temporaries.
|
||||
# (The pure-torch fallback below upcast both halves to fp32 and
|
||||
# allocated ~8 temporaries per call, blowing up MoE memory.)
|
||||
# Both betas come from FusedMoEConfig; a missing beta means the caller
|
||||
# bypassed the config plumbing, so fail rather than silently use 1.0.
|
||||
# linear_beta is genuinely optional: <= 0 signals "unset" to the kernel
|
||||
# (up passed through), matching SituAndMul(linear_beta=None).
|
||||
assert activation_situ_beta is not None, (
|
||||
"SITU requires activation_situ_beta from FusedMoEConfig"
|
||||
)
|
||||
torch.ops._C.situ_and_mul(
|
||||
output,
|
||||
input,
|
||||
activation_situ_beta,
|
||||
-1.0
|
||||
if activation_situ_linear_beta is None
|
||||
else activation_situ_linear_beta,
|
||||
)
|
||||
elif activation == MoEActivation.SWIGLUOAI:
|
||||
torch.ops._C.swigluoai_and_mul(output, input)
|
||||
elif activation == MoEActivation.SWIGLUOAI_UNINTERLEAVE:
|
||||
|
||||
@@ -1314,6 +1314,10 @@ class FusedMoEConfig:
|
||||
swiglu_alpha: float | None = None
|
||||
swiglu_beta: float | None = None
|
||||
|
||||
# SituGLU parameters used by Kimi sit(u/v2) activations.
|
||||
activation_situ_beta: float | None = None
|
||||
activation_situ_linear_beta: float | None = None
|
||||
|
||||
max_capture_size: int = 0
|
||||
|
||||
# Set by __post_init__
|
||||
|
||||
@@ -444,7 +444,14 @@ class DeepGemmFP4Experts(mk.FusedMoEExpertsModular):
|
||||
|
||||
@staticmethod
|
||||
def _supports_activation(activation: MoEActivation) -> bool:
|
||||
return activation in [MoEActivation.SILU, MoEActivation.SWIGLUSTEP]
|
||||
# SILU has fused gate+mul+quant kernels; SWIGLUSTEP/SITU take the
|
||||
# general path (activation applied via self.activation, which forwards
|
||||
# the situ betas, then FP8 requant).
|
||||
return activation in [
|
||||
MoEActivation.SILU,
|
||||
MoEActivation.SWIGLUSTEP,
|
||||
MoEActivation.SITU,
|
||||
]
|
||||
|
||||
@staticmethod
|
||||
def _supports_parallel_config(moe_parallel_config: FusedMoEParallelConfig) -> bool:
|
||||
@@ -488,16 +495,15 @@ class DeepGemmFP4Experts(mk.FusedMoEExpertsModular):
|
||||
M_sum, N = input.size()
|
||||
activation_out_dim = self.adjust_N_for_activation(N, activation)
|
||||
|
||||
if scale_fmt == DeepGemmQuantScaleFMT.UE8M0:
|
||||
assert activation == MoEActivation.SILU
|
||||
return fused_silu_mul_fp8_quant_packed(
|
||||
input=input,
|
||||
output_q=output,
|
||||
group_size=block_k,
|
||||
clamp_limit=self.gemm1_clamp_limit,
|
||||
)
|
||||
|
||||
if activation == MoEActivation.SILU:
|
||||
# Fused gate+mul+quant kernels for the common SILU case.
|
||||
if scale_fmt == DeepGemmQuantScaleFMT.UE8M0:
|
||||
return fused_silu_mul_fp8_quant_packed(
|
||||
input=input,
|
||||
output_q=output,
|
||||
group_size=block_k,
|
||||
clamp_limit=self.gemm1_clamp_limit,
|
||||
)
|
||||
use_ue8m0 = scale_fmt == DeepGemmQuantScaleFMT.FLOAT32_CEIL_UE8M0
|
||||
return silu_mul_per_token_group_quant_fp8_colmajor(
|
||||
input=input,
|
||||
@@ -506,12 +512,24 @@ class DeepGemmFP4Experts(mk.FusedMoEExpertsModular):
|
||||
clamp_limit=self.gemm1_clamp_limit,
|
||||
)
|
||||
|
||||
# General gated activations (SWIGLUSTEP, SITU): apply the activation
|
||||
# (self.activation forwards the situ betas from moe_config) then
|
||||
# FP8-requant into the layout DeepGEMM expects for this scale format.
|
||||
act_out = torch.empty(
|
||||
(M_sum, activation_out_dim), dtype=input.dtype, device=input.device
|
||||
)
|
||||
self.activation(activation, act_out, input)
|
||||
if scale_fmt == DeepGemmQuantScaleFMT.UE8M0:
|
||||
return per_token_group_quant_fp8_packed_for_deepgemm(
|
||||
act_out, block_k, use_ue8m0=True, out_q=output
|
||||
)
|
||||
use_ue8m0 = scale_fmt == DeepGemmQuantScaleFMT.FLOAT32_CEIL_UE8M0
|
||||
return per_token_group_quant_fp8(
|
||||
act_out, block_k, column_major_scales=True, out_q=output
|
||||
act_out,
|
||||
block_k,
|
||||
column_major_scales=True,
|
||||
out_q=output,
|
||||
use_ue8m0=use_ue8m0,
|
||||
)
|
||||
|
||||
def apply(
|
||||
|
||||
@@ -93,6 +93,8 @@ def _fused_marlin_moe(
|
||||
clamp_limit: float | None = None,
|
||||
gemm1_alpha: float = 1.0,
|
||||
gemm1_beta: float = 0.0,
|
||||
activation_situ_beta: float | None = None,
|
||||
activation_situ_linear_beta: float | None = None,
|
||||
) -> torch.Tensor:
|
||||
assert hidden_states.ndim == 2
|
||||
M, K = hidden_states.size()
|
||||
@@ -171,6 +173,8 @@ def _fused_marlin_moe(
|
||||
beta=gemm1_beta,
|
||||
topk_ids=topk_ids,
|
||||
expert_map=expert_map,
|
||||
activation_situ_beta=activation_situ_beta,
|
||||
activation_situ_linear_beta=activation_situ_linear_beta,
|
||||
)
|
||||
|
||||
if output is None:
|
||||
@@ -256,6 +260,8 @@ def fused_marlin_moe(
|
||||
clamp_limit: float | None = None,
|
||||
gemm1_alpha: float = 1.0,
|
||||
gemm1_beta: float = 0.0,
|
||||
activation_situ_beta: float | None = None,
|
||||
activation_situ_linear_beta: float | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
This function computes a Mixture of Experts (MoE) layer using two sets of
|
||||
@@ -357,6 +363,8 @@ def fused_marlin_moe(
|
||||
num_tokens_post_padded=num_tokens_post_padded,
|
||||
activation=activation,
|
||||
activation_func=activation_func,
|
||||
activation_situ_beta=activation_situ_beta,
|
||||
activation_situ_linear_beta=activation_situ_linear_beta,
|
||||
input_global_scale1=input_global_scale1,
|
||||
input_global_scale2=input_global_scale2,
|
||||
global_scale1=global_scale1,
|
||||
@@ -423,6 +431,9 @@ def batched_fused_marlin_moe(
|
||||
clamp_limit: float | None = None,
|
||||
gemm1_alpha: float = 1.0,
|
||||
gemm1_beta: float = 0.0,
|
||||
activation_func: Callable[..., None] = apply_moe_activation,
|
||||
activation_situ_beta: float | None = None,
|
||||
activation_situ_linear_beta: float | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
This function massages the inputs so the batched hidden_states can be
|
||||
@@ -530,6 +541,9 @@ def batched_fused_marlin_moe(
|
||||
quant_type=quant_type,
|
||||
apply_router_weight_on_input=apply_router_weight_on_input,
|
||||
activation=activation,
|
||||
activation_func=activation_func,
|
||||
activation_situ_beta=activation_situ_beta,
|
||||
activation_situ_linear_beta=activation_situ_linear_beta,
|
||||
expert_map=expert_map,
|
||||
block_size_m=block_size_m,
|
||||
sorted_token_ids=sorted_token_ids,
|
||||
@@ -645,6 +659,7 @@ class MarlinExpertsBase(mk.FusedMoEExpertsModular):
|
||||
MoEActivation.SILU,
|
||||
MoEActivation.GELU,
|
||||
MoEActivation.GELU_TANH,
|
||||
MoEActivation.SITU,
|
||||
MoEActivation.SWIGLUOAI,
|
||||
MoEActivation.SWIGLUOAI_UNINTERLEAVE,
|
||||
MoEActivation.SWIGLUSTEP,
|
||||
@@ -793,6 +808,10 @@ class MarlinExperts(LoRAExpertsMixin, MarlinExpertsBase):
|
||||
global_num_experts=global_num_experts,
|
||||
activation=activation,
|
||||
activation_func=self.activation,
|
||||
activation_situ_beta=self.moe_config.activation_situ_beta,
|
||||
activation_situ_linear_beta=(
|
||||
self.moe_config.activation_situ_linear_beta
|
||||
),
|
||||
moe_sum=self.moe_sum,
|
||||
expert_map=expert_map,
|
||||
output=output,
|
||||
@@ -833,6 +852,8 @@ class MarlinExperts(LoRAExpertsMixin, MarlinExpertsBase):
|
||||
beta: float = 0.0,
|
||||
topk_ids: torch.Tensor | None = None,
|
||||
expert_map: torch.Tensor | None = None,
|
||||
activation_situ_beta: float | None = None,
|
||||
activation_situ_linear_beta: float | None = None,
|
||||
) -> None:
|
||||
# act_input = intermediate_cache1 (M*topk, 2N for gated)
|
||||
# act_output = intermediate_cache2 (M*topk, N)
|
||||
@@ -871,6 +892,8 @@ class MarlinExperts(LoRAExpertsMixin, MarlinExpertsBase):
|
||||
beta=beta,
|
||||
topk_ids=topk_ids,
|
||||
expert_map=expert_map,
|
||||
activation_situ_beta=activation_situ_beta,
|
||||
activation_situ_linear_beta=activation_situ_linear_beta,
|
||||
)
|
||||
lora_state["cache2"] = act_output
|
||||
|
||||
@@ -918,6 +941,8 @@ class MarlinExperts(LoRAExpertsMixin, MarlinExpertsBase):
|
||||
global_num_experts=global_num_experts,
|
||||
activation=activation,
|
||||
activation_func=activation_with_lora,
|
||||
activation_situ_beta=self.moe_config.activation_situ_beta,
|
||||
activation_situ_linear_beta=self.moe_config.activation_situ_linear_beta,
|
||||
moe_sum=moe_sum_with_lora,
|
||||
expert_map=expert_map,
|
||||
output=output,
|
||||
@@ -1021,6 +1046,41 @@ class BatchedMarlinExperts(MarlinExpertsBase):
|
||||
apply_router_weight_on_input: bool,
|
||||
):
|
||||
assert expert_tokens_meta is not None, "Num valid tokens per batch is required"
|
||||
|
||||
def activation_func(
|
||||
act: MoEActivation,
|
||||
act_output: torch.Tensor,
|
||||
act_input: torch.Tensor,
|
||||
*,
|
||||
activation_situ_beta: float | None = None,
|
||||
activation_situ_linear_beta: float | None = None,
|
||||
**kwargs,
|
||||
) -> None:
|
||||
if act != MoEActivation.SITU:
|
||||
self.activation(
|
||||
act,
|
||||
act_output,
|
||||
act_input,
|
||||
activation_situ_beta=activation_situ_beta,
|
||||
activation_situ_linear_beta=activation_situ_linear_beta,
|
||||
**kwargs,
|
||||
)
|
||||
return
|
||||
|
||||
num_experts, max_num_tokens = hidden_states.shape[:2]
|
||||
beta = activation_situ_beta
|
||||
linear_beta = activation_situ_linear_beta
|
||||
assert beta is not None, (
|
||||
"SITU requires activation_situ_beta from FusedMoEConfig"
|
||||
)
|
||||
torch.ops._C.masked_situ_and_mul(
|
||||
act_output.view(num_experts, max_num_tokens, -1),
|
||||
act_input.view(num_experts, max_num_tokens, -1),
|
||||
expert_tokens_meta.expert_num_tokens,
|
||||
beta,
|
||||
-1.0 if linear_beta is None else linear_beta,
|
||||
)
|
||||
|
||||
return batched_fused_marlin_moe(
|
||||
hidden_states=hidden_states,
|
||||
expert_num_tokens=expert_tokens_meta.expert_num_tokens,
|
||||
@@ -1051,4 +1111,7 @@ class BatchedMarlinExperts(MarlinExpertsBase):
|
||||
clamp_limit=self.gemm1_clamp_limit,
|
||||
gemm1_alpha=self.gemm1_alpha,
|
||||
gemm1_beta=self.gemm1_beta,
|
||||
activation_func=activation_func,
|
||||
activation_situ_beta=self.moe_config.activation_situ_beta,
|
||||
activation_situ_linear_beta=self.moe_config.activation_situ_linear_beta,
|
||||
)
|
||||
|
||||
@@ -79,6 +79,36 @@ class TrtLlmMxfp4ExpertsBase:
|
||||
else:
|
||||
self.gemm1_clamp_limit = None
|
||||
|
||||
# SITU (SituGLU) TRTLLM-Gen kernel computes
|
||||
# left = alpha * tanh(x0 / alpha) * sigmoid(x0) # gate (x0)
|
||||
# right = beta * tanh(x1 / beta) # up (x1)
|
||||
# which matches vLLM's situ_and_mul with (beta, linear_beta), so map
|
||||
# situ beta -> gatedActAlpha (gemm1_alpha) and situ linear_beta ->
|
||||
# gatedActBeta (gemm1_beta). Both must be > 0.
|
||||
if moe_config.activation == MoEActivation.SITU:
|
||||
situ_beta = moe_config.activation_situ_beta
|
||||
situ_linear_beta = moe_config.activation_situ_linear_beta
|
||||
assert situ_beta is not None and situ_beta > 0, (
|
||||
"SITU requires activation_situ_beta > 0"
|
||||
)
|
||||
assert situ_linear_beta is not None and situ_linear_beta > 0, (
|
||||
"TRTLLM SiTuGlu requires activation_situ_linear_beta > 0 "
|
||||
"(the private cubin has no up-passthrough path)"
|
||||
)
|
||||
self.gemm1_alpha = torch.full(
|
||||
(self.local_num_experts,),
|
||||
float(situ_beta),
|
||||
dtype=torch.float32,
|
||||
device=device,
|
||||
)
|
||||
self.gemm1_beta = torch.full(
|
||||
(self.local_num_experts,),
|
||||
float(situ_linear_beta),
|
||||
dtype=torch.float32,
|
||||
device=device,
|
||||
)
|
||||
self.gemm1_clamp_limit = None
|
||||
|
||||
self.max_capture_size = moe_config.max_capture_size
|
||||
|
||||
@staticmethod
|
||||
@@ -103,7 +133,19 @@ class TrtLlmMxfp4ExpertsBase:
|
||||
|
||||
@staticmethod
|
||||
def _supports_activation(activation: MoEActivation) -> bool:
|
||||
return activation in (MoEActivation.SWIGLUOAI, MoEActivation.SILU)
|
||||
return activation in (
|
||||
MoEActivation.SWIGLUOAI,
|
||||
MoEActivation.SILU,
|
||||
MoEActivation.SITU,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _flashinfer_activation_type(activation: MoEActivation) -> int:
|
||||
from flashinfer.fused_moe.core import ActivationType
|
||||
|
||||
if activation == MoEActivation.SITU:
|
||||
return ActivationType.Situ.value
|
||||
return ActivationType.Swiglu.value
|
||||
|
||||
@staticmethod
|
||||
def activation_format() -> mk.FusedMoEActivationFormat:
|
||||
@@ -142,6 +184,7 @@ class TrtLlmMxfp4ExpertsMonolithic(
|
||||
activation_key: QuantKey | None,
|
||||
) -> bool:
|
||||
return routing_method in [
|
||||
RoutingMethodType.DeepSeekV3,
|
||||
RoutingMethodType.Renormalize,
|
||||
RoutingMethodType.RenormalizeNaive,
|
||||
]
|
||||
@@ -192,8 +235,8 @@ class TrtLlmMxfp4ExpertsMonolithic(
|
||||
device=hidden_states.device,
|
||||
)
|
||||
trtllm_fp4_block_scale_moe(
|
||||
routing_logits=router_logits.to(torch.bfloat16),
|
||||
routing_bias=None,
|
||||
routing_logits=router_logits,
|
||||
routing_bias=e_score_correction_bias,
|
||||
hidden_states=x_quant,
|
||||
hidden_states_scale=x_scale,
|
||||
gemm1_weights=w1,
|
||||
@@ -210,14 +253,15 @@ class TrtLlmMxfp4ExpertsMonolithic(
|
||||
output2_scale_scalar=None,
|
||||
num_experts=global_num_experts,
|
||||
top_k=self.topk,
|
||||
n_group=None,
|
||||
topk_group=None,
|
||||
n_group=(num_expert_group or 0),
|
||||
topk_group=(topk_group or 0),
|
||||
intermediate_size=self.intermediate_size_per_partition,
|
||||
local_expert_offset=self.ep_rank * self.local_num_experts,
|
||||
local_num_experts=self.local_num_experts,
|
||||
routed_scaling_factor=None,
|
||||
routed_scaling_factor=routed_scaling_factor,
|
||||
routing_method_type=self.routing_method_type,
|
||||
do_finalize=True,
|
||||
activation_type=self._flashinfer_activation_type(activation),
|
||||
tune_max_num_tokens=max(self.max_capture_size, 1),
|
||||
output=output,
|
||||
routing_replay_out=routing_replay_out,
|
||||
@@ -338,6 +382,7 @@ class TrtLlmMxfp4ExpertsModular(TrtLlmMxfp4ExpertsBase, mk.FusedMoEExpertsModula
|
||||
"routing_method_type": RoutingMethodType.Renormalize,
|
||||
"do_finalize": True,
|
||||
"enable_pdl": True,
|
||||
"activation_type": self._flashinfer_activation_type(activation),
|
||||
"output": output,
|
||||
"tune_max_num_tokens": max(self.max_capture_size, 1),
|
||||
}
|
||||
|
||||
@@ -110,6 +110,27 @@ class TrtLlmNvFp4ExpertsBase:
|
||||
self.gemm1_alpha = None
|
||||
self.gemm1_beta = None
|
||||
|
||||
# SITU (Kimi SituGLU) TRTLLM-Gen kernel computes
|
||||
# left=alpha*tanh(x0/alpha)*sigmoid(x0), right=beta*tanh(x1/beta),
|
||||
# matching vLLM's situ_and_mul, so map situ beta -> gatedActAlpha
|
||||
# (gemm1_alpha) and situ linear_beta -> gatedActBeta (gemm1_beta).
|
||||
# These operate on the dequantized gate/up, so they are NOT folded by
|
||||
# g1_alphas in process_weights_after_loading.
|
||||
self.is_situ = moe_config.activation == MoEActivation.SITU
|
||||
if self.is_situ:
|
||||
situ_beta = moe_config.activation_situ_beta
|
||||
situ_linear_beta = moe_config.activation_situ_linear_beta
|
||||
assert situ_beta is not None and situ_beta > 0, (
|
||||
"SITU requires activation_situ_beta > 0"
|
||||
)
|
||||
assert situ_linear_beta is not None and situ_linear_beta > 0, (
|
||||
"TRTLLM SiTuGlu requires activation_situ_linear_beta > 0 "
|
||||
"(the private cubin has no up-passthrough path)"
|
||||
)
|
||||
self.gemm1_alpha = _per_expert(situ_beta)
|
||||
self.gemm1_beta = _per_expert(situ_linear_beta)
|
||||
self.gemm1_clamp_limit = None
|
||||
|
||||
logger.debug_once(
|
||||
"activation=%s, gemm1_alpha=%s, gemm1_beta=%s, gemm1_clamp_limit=%s",
|
||||
moe_config.activation,
|
||||
@@ -142,7 +163,10 @@ class TrtLlmNvFp4ExpertsBase:
|
||||
# (via the in-place mul above) and never changes again, so this is a
|
||||
# static, per-expert constant. Register on the layer so EPLB
|
||||
# rearranges it alongside the other expert tensors.
|
||||
if self.gemm1_clamp_limit is not None:
|
||||
# SITU alpha/beta act on the dequantized gate/up (tanh clamps), not the
|
||||
# raw GEMM1 accumulator, so they are registered as-is without the
|
||||
# g1_alphas fold used by the SwiGLU-OAI clamp/beta below.
|
||||
if self.gemm1_clamp_limit is not None and not self.is_situ:
|
||||
gemm1_clamp_limit = self.gemm1_clamp_limit / self.quant_config.g1_alphas
|
||||
layer.register_parameter(
|
||||
"gemm1_clamp_limit",
|
||||
@@ -155,7 +179,11 @@ class TrtLlmNvFp4ExpertsBase:
|
||||
# raw. Register both on the layer so EPLB rearranges them with the
|
||||
# other per-expert tensors.
|
||||
if self.gemm1_beta is not None:
|
||||
gemm1_beta = self.gemm1_beta / self.quant_config.g1_alphas
|
||||
gemm1_beta = (
|
||||
self.gemm1_beta
|
||||
if self.is_situ
|
||||
else self.gemm1_beta / self.quant_config.g1_alphas
|
||||
)
|
||||
layer.register_parameter(
|
||||
"gemm1_beta",
|
||||
torch.nn.Parameter(gemm1_beta, requires_grad=False),
|
||||
@@ -197,13 +225,15 @@ class TrtLlmNvFp4ExpertsBase:
|
||||
|
||||
@staticmethod
|
||||
def _supports_activation(activation: MoEActivation) -> bool:
|
||||
"""Supports SiLU, RELU^2 non-gated, GELU, and clamped SwiGLU-OAI."""
|
||||
"""Supports SiLU, RELU^2 non-gated, GELU, clamped SwiGLU-OAI, and the
|
||||
private SituGLU (SITU) cubin."""
|
||||
return activation in [
|
||||
MoEActivation.SILU,
|
||||
MoEActivation.RELU2_NO_MUL,
|
||||
MoEActivation.GELU,
|
||||
MoEActivation.GELU_TANH,
|
||||
MoEActivation.SWIGLUOAI_UNINTERLEAVE,
|
||||
MoEActivation.SITU,
|
||||
]
|
||||
|
||||
@staticmethod
|
||||
|
||||
@@ -120,6 +120,8 @@ def FusedMoE(
|
||||
swiglu_limit: float | None = None,
|
||||
swiglu_alpha: float | None = None,
|
||||
swiglu_beta: float | None = None,
|
||||
activation_situ_beta: float | None = None,
|
||||
activation_situ_linear_beta: float | None = None,
|
||||
e_score_correction_bias: torch.Tensor | None = None,
|
||||
apply_router_weight_on_input: bool = False,
|
||||
activation: str = "silu",
|
||||
@@ -178,6 +180,8 @@ def FusedMoE(
|
||||
scoring_func: Scoring function for routing ("softmax" or others)
|
||||
routed_scaling_factor: Scaling factor applied to topk_weights or output
|
||||
swiglu_limit: SwiGLU activation limit
|
||||
activation_situ_beta: SituGLU activation beta
|
||||
activation_situ_linear_beta: SituGLU linear beta
|
||||
e_score_correction_bias: Expert score correction bias tensor
|
||||
apply_router_weight_on_input: Whether to apply router weights on input
|
||||
activation: Activation function name ("silu", "gelu", etc.)
|
||||
@@ -352,6 +356,8 @@ def FusedMoE(
|
||||
swiglu_limit=swiglu_limit,
|
||||
swiglu_alpha=swiglu_alpha,
|
||||
swiglu_beta=swiglu_beta,
|
||||
activation_situ_beta=activation_situ_beta,
|
||||
activation_situ_linear_beta=activation_situ_linear_beta,
|
||||
max_capture_size=vllm_config.compilation_config.max_cudagraph_capture_size,
|
||||
skip_final_all_reduce=skip_final_all_reduce,
|
||||
)
|
||||
|
||||
@@ -890,6 +890,8 @@ class FusedMoEExpertsModular(FusedMoEExperts):
|
||||
beta: float = 0.0,
|
||||
topk_ids: torch.Tensor | None = None,
|
||||
expert_map: torch.Tensor | None = None,
|
||||
activation_situ_beta: float | None = None,
|
||||
activation_situ_linear_beta: float | None = None,
|
||||
) -> None:
|
||||
apply_moe_activation(
|
||||
activation,
|
||||
@@ -900,6 +902,16 @@ class FusedMoEExpertsModular(FusedMoEExperts):
|
||||
beta=beta,
|
||||
topk_ids=topk_ids,
|
||||
expert_map=expert_map,
|
||||
activation_situ_beta=(
|
||||
self.moe_config.activation_situ_beta
|
||||
if activation_situ_beta is None
|
||||
else activation_situ_beta
|
||||
),
|
||||
activation_situ_linear_beta=(
|
||||
self.moe_config.activation_situ_linear_beta
|
||||
if activation_situ_linear_beta is None
|
||||
else activation_situ_linear_beta
|
||||
),
|
||||
)
|
||||
|
||||
@abstractmethod
|
||||
@@ -1307,6 +1319,14 @@ class FusedMoEKernelModularImpl:
|
||||
activation,
|
||||
)
|
||||
|
||||
use_output_alias = (
|
||||
output_alias is not None
|
||||
and output_alias.shape == fused_out.shape
|
||||
and output_alias.dtype == fused_out.dtype
|
||||
and output_alias.device == fused_out.device
|
||||
and output_alias.is_contiguous()
|
||||
)
|
||||
|
||||
# If caller's output buffer already matches fused_out shape/dtype, alias
|
||||
# to skip the redundant copy in TopKWeightAndReduceNoOP.apply downstream.
|
||||
# This eliminates ~94% of __amd_rocclr_copyBuffer events (Copy 2 of the
|
||||
@@ -1314,15 +1334,10 @@ class FusedMoEKernelModularImpl:
|
||||
if current_platform.is_rocm():
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
|
||||
if (
|
||||
rocm_aiter_ops.is_fused_moe_enabled()
|
||||
and output_alias is not None
|
||||
and output_alias.shape == fused_out.shape
|
||||
and output_alias.dtype == fused_out.dtype
|
||||
and output_alias.device == fused_out.device
|
||||
and output_alias.is_contiguous()
|
||||
):
|
||||
if use_output_alias and rocm_aiter_ops.is_fused_moe_enabled():
|
||||
fused_out = output_alias
|
||||
elif use_output_alias:
|
||||
fused_out = output_alias
|
||||
|
||||
self.fused_experts.apply(
|
||||
output=fused_out,
|
||||
|
||||
@@ -0,0 +1,255 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import torch
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.config import get_current_vllm_config
|
||||
from vllm.distributed import tensor_model_parallel_all_reduce
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.fused_allreduce_gemma_rms_norm import (
|
||||
_AR_RESIDUAL_RMS_NORM,
|
||||
_can_use_flashinfer,
|
||||
flashinfer_trtllm_fused_allreduce_norm,
|
||||
)
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.utils.torch_utils import aux_stream, current_stream
|
||||
|
||||
from .moe_runner import MoERunner, _unpack
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class LatentMoERunner(MoERunner):
|
||||
"""MoE runner for latent MoE with a replicated routed up-projection.
|
||||
|
||||
Fused path (tp>1, un-reduced combine output, shared expert, no SP):
|
||||
concatenates the un-reduced latent partial (dim d) and the un-reduced
|
||||
shared partial (dim D) into one contiguous buffer, all-reduces once, then
|
||||
splits. The latent half is normed and up-projected locally (replicated
|
||||
up-proj -> full hidden), and the shared add folds into the GEMM epilogue
|
||||
(``torch.addmm``). One collective total, no post-reduction communication.
|
||||
|
||||
Native path: the replicated up-proj produces the full hidden dim on every
|
||||
rank, so the base runner combines routed + shared correctly at any TP size
|
||||
(using two collectives instead of the fused path's one).
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*args,
|
||||
enable_k3_latent_moe_tail_fusion: bool = False,
|
||||
**kwargs,
|
||||
) -> None:
|
||||
super().__init__(*args, **kwargs)
|
||||
self.enable_k3_latent_moe_tail_fusion = enable_k3_latent_moe_tail_fusion
|
||||
use_fused_path = self._use_fused_path()
|
||||
if (
|
||||
self.enable_k3_latent_moe_tail_fusion
|
||||
and use_fused_path
|
||||
and self.moe_config.tp_size not in (8, 16)
|
||||
):
|
||||
logger.warning_once(
|
||||
"K3 latent-MoE tail fusion currently supports TP=8 and TP=16, "
|
||||
"but TP=%d is configured. Falling back to the default path.",
|
||||
self.moe_config.tp_size,
|
||||
)
|
||||
self.enable_k3_latent_moe_tail_fusion = False
|
||||
|
||||
if self.enable_k3_latent_moe_tail_fusion and use_fused_path:
|
||||
vllm_config = get_current_vllm_config()
|
||||
if vllm_config.parallel_config.use_ubatching:
|
||||
raise ValueError(
|
||||
"K3 latent-MoE tail fusion does not support DBO or ubatching."
|
||||
)
|
||||
if vllm_config.model_config.enable_sleep_mode:
|
||||
raise ValueError(
|
||||
"K3 latent-MoE tail fusion does not support sleep mode."
|
||||
)
|
||||
transform = self.routed_output_transform
|
||||
assert transform is not None
|
||||
norm = transform.norm
|
||||
assert norm is not None
|
||||
from vllm.models.kimi_k3.nvidia.ops.latent_moe_tail import (
|
||||
KimiK3LatentMoETailOp,
|
||||
)
|
||||
|
||||
op = KimiK3LatentMoETailOp.initialize(
|
||||
hidden_size=transform.up_proj.weight.shape[0],
|
||||
latent_size=norm.weight.shape[0],
|
||||
dtype=norm.weight.dtype,
|
||||
device=norm.weight.device,
|
||||
rms_eps=norm.variance_epsilon,
|
||||
)
|
||||
self._k3_latent_moe_tail_op = op
|
||||
|
||||
def _get_zero_residual(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
||||
"""Read-only zero ``residual_in`` for the fused AR+RMSNorm kernel.
|
||||
|
||||
flashinfer requires a residual buffer even when there is no residual to
|
||||
add.
|
||||
"""
|
||||
numel = hidden_states.numel()
|
||||
buf = getattr(self, "_zero_residual", None)
|
||||
if (
|
||||
buf is None
|
||||
or buf.dtype != hidden_states.dtype
|
||||
or buf.device != hidden_states.device
|
||||
or buf.numel() < numel
|
||||
):
|
||||
buf = torch.zeros(
|
||||
numel, dtype=hidden_states.dtype, device=hidden_states.device
|
||||
)
|
||||
self._zero_residual = buf
|
||||
return buf[:numel].view_as(hidden_states)
|
||||
|
||||
def _use_fused_path(self) -> bool:
|
||||
# The fused path merges the latent and shared reductions into one
|
||||
# all-reduce, so it needs actual TP parallelism, a shared expert (to
|
||||
# concat), an un-reduced combine output, and no sequence parallelism.
|
||||
return (
|
||||
self.moe_config.tp_size > 1
|
||||
and self._shared_experts is not None
|
||||
and not self._fused_output_is_reduced
|
||||
and not self.moe_config.is_sequence_parallel
|
||||
)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
router_logits: torch.Tensor,
|
||||
input_ids: torch.Tensor | None = None,
|
||||
shared_experts_input: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
if self._use_fused_path():
|
||||
return self._fused_forward(
|
||||
hidden_states, router_logits, input_ids, shared_experts_input
|
||||
)
|
||||
return super().forward(
|
||||
hidden_states, router_logits, input_ids, shared_experts_input
|
||||
)
|
||||
|
||||
def _fused_forward(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
router_logits: torch.Tensor,
|
||||
input_ids: torch.Tensor | None = None,
|
||||
shared_experts_input: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
# When the caller pre-applies the routed input transform outside the
|
||||
# runner (e.g. to overlap it on a separate stream), it passes the
|
||||
# already-transformed routed input as ``hidden_states`` and the original
|
||||
# hidden states as ``shared_experts_input``; skip the transform then.
|
||||
if shared_experts_input is None:
|
||||
hidden_states, shared_experts_input = self.apply_routed_input_transform(
|
||||
hidden_states
|
||||
)
|
||||
hidden_states, og_hidden_dim_pre_xform, og_hidden_dim_post_xform = (
|
||||
self._maybe_pad_hidden_states(
|
||||
shared_experts_input,
|
||||
hidden_states,
|
||||
)
|
||||
)
|
||||
|
||||
result = self._forward_entry(
|
||||
hidden_states,
|
||||
router_logits,
|
||||
shared_experts_input,
|
||||
input_ids,
|
||||
self._encode_layer_name(),
|
||||
self.moe_config.hidden_dim_unpadded
|
||||
if self._quant_method.has_unpadded_output
|
||||
else 0,
|
||||
)
|
||||
|
||||
shared_output, fused_output = _unpack(result)
|
||||
assert shared_output is not None
|
||||
|
||||
if og_hidden_dim_pre_xform is not None:
|
||||
fused_output = fused_output[..., :og_hidden_dim_pre_xform]
|
||||
|
||||
transform = self.routed_output_transform
|
||||
assert transform is not None
|
||||
|
||||
if self.enable_k3_latent_moe_tail_fusion:
|
||||
op = self._k3_latent_moe_tail_op
|
||||
if 0 < fused_output.shape[0] <= op.contract.max_num_tokens:
|
||||
norm = transform.norm
|
||||
assert norm is not None
|
||||
result = op(
|
||||
fused_output,
|
||||
shared_output,
|
||||
norm.weight,
|
||||
transform.up_proj.weight,
|
||||
)
|
||||
result = self._maybe_reduce_final_output(
|
||||
result, og_hidden_dim_post_xform, output_is_reduced=True
|
||||
)
|
||||
return self._maybe_add_zero_expert_output(result)
|
||||
|
||||
fused_latent = None
|
||||
if transform.norm is not None:
|
||||
fused_latent = self.allreduce_norm_latent_out(fused_output, transform.norm)
|
||||
else:
|
||||
fused_latent = tensor_model_parallel_all_reduce(fused_output)
|
||||
|
||||
shared_expert_stream = (
|
||||
aux_stream()
|
||||
if shared_output.size(0) <= envs.VLLM_SHARED_EXPERTS_STREAM_TOKEN_THRESHOLD
|
||||
else None
|
||||
)
|
||||
if shared_expert_stream is not None:
|
||||
# overlap shared expert allreduce with latent up_proj
|
||||
main = current_stream()
|
||||
shared_output.record_stream(shared_expert_stream)
|
||||
shared_expert_stream.wait_stream(main)
|
||||
with torch.cuda.stream(shared_expert_stream):
|
||||
shared_output = tensor_model_parallel_all_reduce(shared_output)
|
||||
result = torch.mm(fused_latent, transform.up_proj.weight.t())
|
||||
main.wait_stream(shared_expert_stream)
|
||||
else:
|
||||
shared_output = tensor_model_parallel_all_reduce(shared_output)
|
||||
result = torch.mm(fused_latent, transform.up_proj.weight.t())
|
||||
result.add_(shared_output)
|
||||
|
||||
# Output is already fully reduced; this only strips padding.
|
||||
result = self._maybe_reduce_final_output(
|
||||
result, og_hidden_dim_post_xform, output_is_reduced=True
|
||||
)
|
||||
|
||||
return self._maybe_add_zero_expert_output(result)
|
||||
|
||||
def allreduce_norm_latent_out(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
norm: RMSNorm,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""All-reduce + add residual + (standard) RMSNorm, fused via flashinfer."""
|
||||
if self.moe_config.tp_size == 1:
|
||||
return norm(hidden_states)
|
||||
|
||||
if flashinfer_trtllm_fused_allreduce_norm is not None:
|
||||
ok, max_token_num = _can_use_flashinfer(
|
||||
hidden_states, self.moe_config.tp_size
|
||||
)
|
||||
if ok:
|
||||
norm_out = torch.empty_like(hidden_states)
|
||||
# With norm_out provided, the kernel writes the new residual
|
||||
# (all_reduce(hidden_states) + residual) into the hidden_states
|
||||
# buffer and the normalized result into norm_out.
|
||||
flashinfer_trtllm_fused_allreduce_norm(
|
||||
allreduce_in=hidden_states,
|
||||
residual=self._get_zero_residual(hidden_states),
|
||||
rms_gamma=norm.weight,
|
||||
rms_eps=norm.variance_epsilon,
|
||||
world_size=self.moe_config.tp_size,
|
||||
weight_bias=0.0,
|
||||
launch_with_pdl=True,
|
||||
fp32_acc=True,
|
||||
max_token_num=max_token_num,
|
||||
pattern_code=_AR_RESIDUAL_RMS_NORM,
|
||||
norm_out=norm_out,
|
||||
)
|
||||
return norm_out
|
||||
|
||||
reduced = tensor_model_parallel_all_reduce(hidden_states)
|
||||
return norm(reduced)
|
||||
@@ -348,7 +348,8 @@ class MoERunner(MoERunnerInterface):
|
||||
return self.routed_experts.quant_method
|
||||
|
||||
def apply_routed_input_transform(
|
||||
self, hidden_states: torch.Tensor
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor | None]:
|
||||
"""Apply transform for routed experts (e.g., latent projection).
|
||||
|
||||
@@ -416,6 +417,7 @@ class MoERunner(MoERunnerInterface):
|
||||
def _maybe_reduce_shared_expert_output(
|
||||
self,
|
||||
shared_output: torch.Tensor | None,
|
||||
fused_output_is_reduced: bool | None = None,
|
||||
) -> torch.Tensor | None:
|
||||
"""All-reduce shared expert output when the combine kernel already
|
||||
reduced fused output.
|
||||
@@ -425,18 +427,43 @@ class MoERunner(MoERunnerInterface):
|
||||
* If we have SP (TP=N, DP=M, EP), there is a separate AG step handled
|
||||
in the model.
|
||||
"""
|
||||
if fused_output_is_reduced is None:
|
||||
fused_output_is_reduced = self._fused_output_is_reduced
|
||||
|
||||
if (
|
||||
shared_output is not None
|
||||
and not self.moe_config.is_sequence_parallel
|
||||
and self._fused_output_is_reduced
|
||||
and fused_output_is_reduced
|
||||
):
|
||||
shared_output = tensor_model_parallel_all_reduce(shared_output)
|
||||
return shared_output
|
||||
|
||||
def _maybe_reduce_routed_output_before_transform(
|
||||
self,
|
||||
fused_output: torch.Tensor,
|
||||
fused_output_is_reduced: bool,
|
||||
) -> tuple[torch.Tensor, bool]:
|
||||
"""All-reduce latent routed output before its output transform.
|
||||
|
||||
Latent MoE output transforms may contain non-linear ops, e.g. RMSNorm.
|
||||
TP partial routed outputs must be summed in latent space before such
|
||||
transforms are applied.
|
||||
"""
|
||||
if (
|
||||
self.routed_output_transform is not None
|
||||
and not self.moe_config.is_sequence_parallel
|
||||
and (self.moe_config.tp_size > 1 or self.moe_config.ep_size > 1)
|
||||
and not fused_output_is_reduced
|
||||
):
|
||||
fused_output = tensor_model_parallel_all_reduce(fused_output)
|
||||
fused_output_is_reduced = True
|
||||
return fused_output, fused_output_is_reduced
|
||||
|
||||
def _maybe_reduce_final_output(
|
||||
self,
|
||||
states: torch.Tensor,
|
||||
trunc_size: int | None,
|
||||
output_is_reduced: bool | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""All-reduce the combined output if needed.
|
||||
|
||||
@@ -455,11 +482,14 @@ class MoERunner(MoERunnerInterface):
|
||||
# We don't need to reduce the final output if:
|
||||
# - We are not running with TP or DP
|
||||
# - The MK already reduced the fused output itself.
|
||||
if output_is_reduced is None:
|
||||
output_is_reduced = self._fused_output_is_reduced
|
||||
|
||||
if (
|
||||
not self.moe_config.is_sequence_parallel
|
||||
and not self.moe_config.skip_final_all_reduce
|
||||
and (self.moe_config.tp_size > 1 or self.moe_config.ep_size > 1)
|
||||
and not self._fused_output_is_reduced
|
||||
and not output_is_reduced
|
||||
):
|
||||
states = tensor_model_parallel_all_reduce(states)
|
||||
|
||||
@@ -643,6 +673,7 @@ class MoERunner(MoERunnerInterface):
|
||||
hidden_states: torch.Tensor,
|
||||
router_logits: torch.Tensor,
|
||||
input_ids: torch.Tensor | None = None,
|
||||
shared_experts_input: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""Invoke the fused moe layer.
|
||||
|
||||
@@ -664,11 +695,16 @@ class MoERunner(MoERunnerInterface):
|
||||
_moe_forward and _moe_forward_shared must be split.
|
||||
"""
|
||||
|
||||
# Apply transform for routed experts (e.g., latent projection
|
||||
# for latent MoE)
|
||||
hidden_states, shared_experts_input = self.apply_routed_input_transform(
|
||||
hidden_states
|
||||
)
|
||||
# Apply transform for routed experts (e.g., latent projection for
|
||||
# latent MoE). When the caller pre-applies the routed input transform
|
||||
# outside the runner (e.g. to overlap it on a separate stream), it
|
||||
# passes the already-transformed routed input as ``hidden_states`` and
|
||||
# the original hidden states as ``shared_experts_input``; skip the
|
||||
# transform in that case so shared experts still see the original input.
|
||||
if shared_experts_input is None:
|
||||
hidden_states, shared_experts_input = self.apply_routed_input_transform(
|
||||
hidden_states
|
||||
)
|
||||
|
||||
# Record before `_maybe_pad_hidden_states` pads activations to match
|
||||
# `moe_config.hidden_dim`, e.g. after `align_trtllm_fp4_moe_hidden_dim_for_fi`
|
||||
@@ -708,9 +744,22 @@ class MoERunner(MoERunnerInterface):
|
||||
if og_hidden_dim_pre_xform is not None:
|
||||
fused_output = fused_output[..., :og_hidden_dim_pre_xform]
|
||||
|
||||
# If combine kernel already reduced fused, reduce shared to match.
|
||||
fused_output_is_reduced = self._fused_output_is_reduced
|
||||
|
||||
# Latent routed output has to be reduced before output transform,
|
||||
# because the transform may include non-linear normalization.
|
||||
fused_output, fused_output_is_reduced = (
|
||||
self._maybe_reduce_routed_output_before_transform(
|
||||
fused_output,
|
||||
fused_output_is_reduced,
|
||||
)
|
||||
)
|
||||
|
||||
# If routed output is already reduced, reduce shared to match.
|
||||
# See note above re: the two all-reduce points.
|
||||
shared_output = self._maybe_reduce_shared_expert_output(shared_output)
|
||||
shared_output = self._maybe_reduce_shared_expert_output(
|
||||
shared_output, fused_output_is_reduced
|
||||
)
|
||||
|
||||
shared_output, fused_output = self._maybe_apply_routed_scale_to_output(
|
||||
shared_output, fused_output
|
||||
@@ -724,7 +773,9 @@ class MoERunner(MoERunnerInterface):
|
||||
else:
|
||||
result = fused_output
|
||||
|
||||
result = self._maybe_reduce_final_output(result, og_hidden_dim_post_xform)
|
||||
result = self._maybe_reduce_final_output(
|
||||
result, og_hidden_dim_post_xform, fused_output_is_reduced
|
||||
)
|
||||
|
||||
return self._maybe_add_zero_expert_output(result)
|
||||
|
||||
|
||||
@@ -36,6 +36,7 @@ class MoERunnerInterface(PluggableLayer, ABC):
|
||||
hidden_states: torch.Tensor,
|
||||
router_logits: torch.Tensor,
|
||||
input_ids: torch.Tensor | None = None,
|
||||
shared_experts_input: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
@@ -1080,6 +1080,7 @@ def causal_conv1d_update(
|
||||
block_idx_last_scheduled_token: torch.Tensor | None = None,
|
||||
initial_state_idx: torch.Tensor | None = None,
|
||||
validate_data=False,
|
||||
out: torch.Tensor | None = None,
|
||||
):
|
||||
"""
|
||||
x: Input tensor which can take the following shapes:
|
||||
@@ -1117,7 +1118,8 @@ def causal_conv1d_update(
|
||||
for example: conv_state_indices = [null_block_id, 1, 20, null_block_id]
|
||||
in this case, the kernel will not process entries at
|
||||
indices 0 and 3
|
||||
out: (batch, dim) or (batch, dim, seqlen) or (num_tokens, dim), same shape as `x`
|
||||
out: optional output tensor with the same shape as `x`. When omitted,
|
||||
the input is overwritten.
|
||||
"""
|
||||
if validate_data:
|
||||
assert null_block_id is not None
|
||||
@@ -1129,10 +1131,22 @@ def causal_conv1d_update(
|
||||
|
||||
original_x_dtype = x.dtype
|
||||
x = x.to(conv_state.dtype)
|
||||
if out is None:
|
||||
out = x
|
||||
else:
|
||||
if out.shape != x.shape:
|
||||
raise ValueError(
|
||||
f"`out` shape {tuple(out.shape)} must match `x` shape {tuple(x.shape)}."
|
||||
)
|
||||
if out.dtype != original_x_dtype or out.device != x.device:
|
||||
raise ValueError(
|
||||
"`out` must have the same dtype and device as the input `x`."
|
||||
)
|
||||
unsqueeze = query_start_loc is None and x.dim() == 2
|
||||
if unsqueeze:
|
||||
# make it (batch, dim, seqlen) with seqlen == 1
|
||||
x = x.unsqueeze(-1)
|
||||
out = out.unsqueeze(-1)
|
||||
if query_start_loc is None:
|
||||
batch, dim, seqlen = x.shape
|
||||
else:
|
||||
@@ -1159,8 +1173,6 @@ def causal_conv1d_update(
|
||||
assert num_cache_lines >= batch
|
||||
assert weight.stride(1) == 1 # Need this
|
||||
|
||||
# adopt the strategy in vLLM that overwrite on 'x' directly, rather than creating a new tensor 'o'
|
||||
out = x
|
||||
stride_w_dim, stride_w_width = weight.stride()
|
||||
|
||||
if query_start_loc is None:
|
||||
|
||||
@@ -0,0 +1,83 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import tl, triton
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _gather_initial_states_kernel(
|
||||
state_ptr,
|
||||
indices_ptr,
|
||||
has_initial_state_ptr,
|
||||
output_ptr,
|
||||
stride_state_batch,
|
||||
stride_indices,
|
||||
stride_has_initial_state,
|
||||
row_size: tl.constexpr,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
launch_pdl: tl.constexpr,
|
||||
):
|
||||
block_idx = tl.program_id(0)
|
||||
batch_idx = tl.program_id(1)
|
||||
offsets = block_idx * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)
|
||||
mask = offsets < row_size
|
||||
|
||||
if launch_pdl:
|
||||
tl.extra.cuda.gdc_wait()
|
||||
tl.extra.cuda.gdc_launch_dependents()
|
||||
|
||||
has_initial_state = tl.load(
|
||||
has_initial_state_ptr + batch_idx * stride_has_initial_state
|
||||
).to(tl.int1)
|
||||
state_idx = tl.load(indices_ptr + batch_idx * stride_indices).to(tl.int64)
|
||||
state_idx = tl.where(has_initial_state, state_idx, 0)
|
||||
values = tl.load(
|
||||
state_ptr + state_idx * stride_state_batch + offsets,
|
||||
mask=mask & has_initial_state,
|
||||
other=0.0,
|
||||
)
|
||||
tl.store(output_ptr + batch_idx * row_size + offsets, values, mask=mask)
|
||||
|
||||
|
||||
def gather_initial_states(
|
||||
state: torch.Tensor,
|
||||
indices: torch.Tensor,
|
||||
has_initial_state: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
"""Gather dense state rows, replacing uninitialized rows with zeros."""
|
||||
assert state.ndim >= 2
|
||||
assert state.is_cuda
|
||||
assert indices.ndim == 1 and has_initial_state.ndim == 1
|
||||
assert indices.shape == has_initial_state.shape
|
||||
assert indices.device == state.device
|
||||
assert has_initial_state.device == state.device
|
||||
assert indices.dtype in (torch.int32, torch.int64)
|
||||
assert has_initial_state.dtype == torch.bool
|
||||
|
||||
row_size = state[0].numel()
|
||||
# Mamba pages may pad stride(0), but each state row remains dense.
|
||||
assert state[0].is_contiguous()
|
||||
output = torch.empty(
|
||||
(indices.numel(), *state.shape[1:]),
|
||||
dtype=state.dtype,
|
||||
device=state.device,
|
||||
)
|
||||
block_size = min(triton.next_power_of_2(row_size), 1024)
|
||||
grid = (triton.cdiv(row_size, block_size), indices.numel())
|
||||
_gather_initial_states_kernel[grid](
|
||||
state,
|
||||
indices,
|
||||
has_initial_state,
|
||||
output,
|
||||
state.stride(0),
|
||||
indices.stride(0),
|
||||
has_initial_state.stride(0),
|
||||
row_size=row_size,
|
||||
BLOCK_SIZE=block_size,
|
||||
num_warps=8,
|
||||
launch_pdl=current_platform.is_arch_support_pdl(),
|
||||
)
|
||||
return output
|
||||
Reference in New Issue
Block a user