[New model] Kimi K3 (#50000)

Signed-off-by: zjy0516 <riverclouds.zhu@qq.com>
Signed-off-by: Isotr0py <Isotr0py@outlook.com>
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
Signed-off-by: Jee Jee Li <jeejeelee@inferact.ai>
Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai>
Signed-off-by: Yongye Zhu <zyy1102000@gmail.com>
Signed-off-by: wzhao18 <wzhao18.sz@gmail.com>
Co-authored-by: Isotr0py <Isotr0py@outlook.com>
Co-authored-by: Thien Tran <gau.nernst@yahoo.com.sg>
Co-authored-by: Bugen Zhao <i@bugenzhao.com>
Co-authored-by: Yongye Zhu <zyy1102000@gmail.com>
Co-authored-by: Ziming Huang <zelda.huanghuang@gmail.com>
Co-authored-by: Roger Wang <hey@rogerw.io>
Co-authored-by: Isotr0py <mozf@inferact.ai>
Co-authored-by: aoshen02 <aoshen02@users.noreply.github.com>
Co-authored-by: Woosuk Kwon <woosuk@inferact.ai>
Co-authored-by: Jee Jee Li <jeejeelee@inferact.ai>
Co-authored-by: aoshen02 <aoshen@inferact.ai>
Co-authored-by: Summer Yang <girasoleyang@gmail.com>
Co-authored-by: Kevin H. Luu <khluu000@gmail.com>
Co-authored-by: Bowen Wang <abmfy@icloud.com>
Co-authored-by: gnovack <novackgm@gmail.com>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai>
Co-authored-by: xiaozhoupy <peiyuanzhou1994@gmail.com>
Co-authored-by: Roy Wang <yasong.wang@inferact.ai>
Co-authored-by: Jeff (Junze) Ma <93145857+majunze2001@users.noreply.github.com>
Co-authored-by: Codex <codex@openai.com>
Co-authored-by: Wei Zhao <51183510+wzhao18@users.noreply.github.com>
This commit is contained in:
Jiangyun Zhu
2026-07-30 03:49:58 -07:00
committed by GitHub
co-authored by Isotr0py Thien Tran Bugen Zhao Yongye Zhu Ziming Huang Roger Wang Isotr0py aoshen02 Woosuk Kwon Jee Jee Li aoshen02 Summer Yang Kevin H. Luu Bowen Wang gnovack Nick Hill Yifan Qiao xiaozhoupy Roy Wang Jeff Ma Codex Wei Zhao
parent 0c64be8873
commit aeeb36b1f1
82 changed files with 2927 additions and 1345 deletions
@@ -272,6 +272,7 @@ from vllm.v1.attention.backends.mla.prefill import (
)
from vllm.v1.attention.backends.utils import (
get_dcp_local_seq_lens,
get_num_attention_heads_from_layers,
split_decodes_and_prefills,
)
from vllm.v1.attention.ops.common import cp_lse_ag_out_ar, cp_lse_ag_out_rs
@@ -375,6 +376,7 @@ class MLAAttention(nn.Module, AttentionLayerBase):
use_sparse: bool = False,
indexer: object | None = None,
topk_indices_buffer: torch.Tensor | None = None,
non_causal_multi_token_decode: bool = False,
**extra_impl_args,
):
super().__init__()
@@ -391,6 +393,7 @@ class MLAAttention(nn.Module, AttentionLayerBase):
self.head_size = kv_lora_rank + qk_rope_head_dim
self.layer_name = prefix
self.indexer = indexer
self.non_causal_multi_token_decode = non_causal_multi_token_decode
self.num_kv_heads = 1
self.qk_head_dim = self.qk_nope_head_dim + self.qk_rope_head_dim
@@ -1123,6 +1126,7 @@ class MLAAttention(nn.Module, AttentionLayerBase):
dtype=kv_cache_dtype,
cache_dtype_str=self.kv_cache_dtype,
kv_quant_mode=get_kv_quant_mode(self.kv_cache_dtype),
non_causal_multi_token_decode=self.non_causal_multi_token_decode,
)
def _v_up_proj(self, x: torch.Tensor, out: torch.Tensor):
@@ -1450,6 +1454,8 @@ class MLACommonMetadata(AttentionMetadata, Generic[D]):
num_decode_tokens: int
num_prefills: int
causal: bool = True
# The dimension of the attention heads
head_dim: int | None = None
@@ -1724,6 +1730,9 @@ class MLACommonMetadataBuilder(AttentionMetadataBuilder[M]):
# speculative decoding is enabled.
query_len_support: ClassVar[QueryLenSupport] = QueryLenSupport.SINGLE_ONLY
# Whether this builder can flatten a non-causal query block into decode rows.
supports_non_causal_multi_token_decode: ClassVar[bool] = False
# The threshold for reordering the batch into decode and prefill requests.
# If > 1, the batch will be reordered such that requests with
# query length <= threshold are classified as decode requests.
@@ -1824,8 +1833,14 @@ class MLACommonMetadataBuilder(AttentionMetadataBuilder[M]):
self.vllm_config = vllm_config
self.device = device
self.use_pcp = parallel_config.prefill_context_parallel_size > 1
self.non_causal_multi_token_decode = getattr(
kv_cache_spec, "non_causal_multi_token_decode", False
)
self.num_heads = self.model_config.get_num_attention_heads(parallel_config)
# A draft cache group can have a different head count from the target.
self.num_heads = get_num_attention_heads_from_layers(
vllm_config, layer_names
) or self.model_config.get_num_attention_heads(parallel_config)
self.mla_dims = get_mla_dims(self.model_config)
self.aot_schedule = current_platform.is_cuda()
@@ -1951,14 +1966,42 @@ class MLACommonMetadataBuilder(AttentionMetadataBuilder[M]):
seq_lens = common_attn_metadata.seq_lens
dcp_local_seq_lens = common_attn_metadata.dcp_local_seq_lens
num_decodes, num_prefills, num_decode_tokens, num_prefill_tokens = (
split_decodes_and_prefills(
common_attn_metadata,
decode_threshold=self.reorder_batch_threshold,
require_uniform=(self.query_len_support != QueryLenSupport.VARLEN),
treat_short_extends_as_decodes=not self.use_pcp,
non_causal_decode = common_attn_metadata.causal is False
if non_causal_decode:
if not (
self.supports_non_causal_multi_token_decode
and self.non_causal_multi_token_decode
):
raise ValueError(
"Non-causal multi-token MLA requires an explicitly supported "
"attention group."
)
query_lens = query_start_loc_cpu[1:] - query_start_loc_cpu[:-1]
num_active_reqs = int(torch.count_nonzero(query_lens > 0))
uniform_active_queries = num_active_reqs > 0 and bool(
torch.all(query_lens[:num_active_reqs] == query_lens[0])
)
trailing_graph_padding = bool(torch.all(query_lens[num_active_reqs:] == 0))
if not (uniform_active_queries and trailing_graph_padding):
raise ValueError(
"Non-causal MLA requires a uniform query block; got query "
f"lengths {query_lens.tolist()}."
)
# Use exact GPU sequence lengths instead of the prefill path's CPU
# context-length upper bounds.
num_decodes = num_reqs
num_prefills = 0
num_decode_tokens = num_tokens
num_prefill_tokens = 0
else:
num_decodes, num_prefills, num_decode_tokens, num_prefill_tokens = (
split_decodes_and_prefills(
common_attn_metadata,
decode_threshold=self.reorder_batch_threshold,
require_uniform=(self.query_len_support != QueryLenSupport.VARLEN),
treat_short_extends_as_decodes=not self.use_pcp,
)
)
)
assert num_decodes + num_prefills == num_reqs
assert num_decode_tokens + num_prefill_tokens == num_tokens
@@ -2049,6 +2092,7 @@ class MLACommonMetadataBuilder(AttentionMetadataBuilder[M]):
num_decodes=num_decodes,
num_decode_tokens=num_decode_tokens,
num_prefills=num_prefills,
causal=not non_causal_decode,
prefill=prefill_metadata,
decode=decode_metadata,
)
@@ -505,7 +505,7 @@ def deepgemm_moe_permute(
dtype=torch.int32,
)
else:
aq_scale_out = torch.empty((M_sum, sf_k), device=device, dtype=torch.float32)
aq_scale_out = torch.zeros((M_sum, sf_k), device=device, dtype=torch.float32)
# DeepGEMM uses negative values in m_indices (here expert_ids) to mark
# completely invalid / padded blocks that should be skipped. We always
@@ -204,6 +204,7 @@ def persistent_masked_m_silu_mul_quant(
dtype=ys_dtype,
device=y.device,
)
y_s.zero_()
ceil_ue8m0 = quant_scale_fmt in [
DeepGemmQuantScaleFMT.FLOAT32_CEIL_UE8M0,
@@ -671,10 +671,10 @@ class MarlinExpertsBase(mk.FusedMoEExpertsModular):
@staticmethod
def _supports_parallel_config(moe_parallel_config: FusedMoEParallelConfig) -> bool:
return not (
moe_parallel_config.use_fi_nvl_two_sided_kernels
or moe_parallel_config.use_fi_nvl_one_sided_kernels
)
# One-sided FI-NVL all2all pairs with MarlinExperts fine (the
# compressed-tensors MXFP4 path runs this exact combo); only the
# two-sided kernels are unsupported here.
return not moe_parallel_config.use_fi_nvl_two_sided_kernels
@property
def quant_type_id(self) -> int:
@@ -1068,11 +1068,8 @@ class BatchedMarlinExperts(MarlinExpertsBase):
return
num_experts, max_num_tokens = hidden_states.shape[:2]
beta = activation_situ_beta
beta = 1.0 if activation_situ_beta is None else activation_situ_beta
linear_beta = activation_situ_linear_beta
assert beta is not None, (
"SITU requires activation_situ_beta from FusedMoEConfig"
)
torch.ops._C.masked_situ_and_mul(
act_output.view(num_experts, max_num_tokens, -1),
act_input.view(num_experts, max_num_tokens, -1),
@@ -5,6 +5,7 @@ from functools import lru_cache
import torch
import vllm.envs as envs
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
from vllm._aiter_ops import rocm_aiter_ops
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
@@ -262,6 +263,8 @@ def rocm_aiter_fused_experts(
elif activation == MoEActivation.SWIGLUOAI_UNINTERLEAVE:
activation_method = rocm_aiter_ops.get_aiter_activation_type("swiglu")
activation_interleave = False
elif activation == MoEActivation.SITU:
activation_method = rocm_aiter_ops.get_aiter_activation_type("situ")
else:
raise ValueError(f"Unsupported activation: {activation}")
@@ -356,10 +359,12 @@ def rocm_aiter_fused_experts(
# https://github.com/ROCm/aiter/blob/v0.1.13.post1/aiter/fused_moe.py#L1099
# TODO: Revisit this once we bump AITER to 0.1.15 with padding fixes
# for CK/FlyDSL MoE GEMM e.g. https://github.com/ROCm/aiter/pull/3401
hidden_pad = hidden_pad // 128 * 128
intermediate_pad = (
intermediate_pad // 64 * 64 * (2 if moe_config.tp_size == 1 else 1)
)
# SITU's A16W4 FlyDSL kernel pads per gate/up half; pass through unrounded.
if activation != MoEActivation.SITU:
hidden_pad = hidden_pad // 128 * 128
intermediate_pad = (
intermediate_pad // 64 * 64 * (2 if moe_config.tp_size == 1 else 1)
)
# https://github.com/ROCm/aiter/pull/3123 specialized the AITER stage1 GEMMs
# for interleaved vs separated gate and up weights.
@@ -370,7 +375,15 @@ def rocm_aiter_fused_experts(
from aiter.ops.flydsl.moe_common import GateMode
gate_mode = ""
if quant_config.use_mxfp4_w4a16:
if activation == MoEActivation.SITU:
# a8w4 (AITER_SITUV2_A8W4=1) uses the gate/up-interleaved (_gui_)
# fp8 flydsl kernels; default a16w4 SiTU stays separated.
gate_mode = (
GateMode.INTERLEAVE.value
if envs.AITER_SITUV2_A8W4
else GateMode.SEPARATED.value
)
elif quant_config.use_mxfp4_w4a16:
gate_mode = GateMode.INTERLEAVE.value
elif activation_interleave is not None:
gate_mode = (
@@ -401,6 +414,8 @@ def rocm_aiter_fused_experts(
bias1=quant_config.w1_bias if quant_config.use_mxfp4_w4a16 else None,
bias2=quant_config.w2_bias if quant_config.use_mxfp4_w4a16 else None,
moe_sorting_dispatch_policy=moe_sorting_dispatch_policy,
beta=moe_config.activation_situ_beta,
linear_beta=moe_config.activation_situ_linear_beta,
)
@@ -15,6 +15,10 @@ from vllm.model_executor.layers.fused_moe.topk_weight_and_reduce import (
TopKWeightAndReduceNoOP,
)
from vllm.model_executor.layers.fused_moe.utils import trtllm_moe_pack_topk_ids_weights
from vllm.model_executor.layers.quantization.utils.flashinfer_utils import (
activation_to_flashinfer_int,
has_flashinfer_situ_activation,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
QuantKey,
kMxfp4Static,
@@ -133,19 +137,16 @@ class TrtLlmMxfp4ExpertsBase:
@staticmethod
def _supports_activation(activation: MoEActivation) -> bool:
if activation == MoEActivation.SITU:
return has_flashinfer_situ_activation()
return activation in (
MoEActivation.SWIGLUOAI,
MoEActivation.SILU,
MoEActivation.SITU,
)
@staticmethod
def _flashinfer_activation_type(activation: MoEActivation) -> int:
from flashinfer.fused_moe.core import ActivationType
if activation == MoEActivation.SITU:
return ActivationType.Situ.value
return ActivationType.Swiglu.value
return activation_to_flashinfer_int(activation)
@staticmethod
def activation_format() -> mk.FusedMoEActivationFormat:
@@ -315,6 +316,75 @@ class TrtLlmMxfp4ExpertsModular(TrtLlmMxfp4ExpertsBase, mk.FusedMoEExpertsModula
output = (M, self.hidden_dim_unpadded)
return (workspace1, workspace2, output)
def _max_supported_tokens(self, top_k: int, global_num_experts: int) -> int:
"""Max tokens per kernel call before the batched-GEMM grid overflows.
The TRTLLM-Gen batched GEMM launches a static grid whose batch (Y)
dimension is ``getMaxNumCtasInBatchDim(num_tokens, top_k, num_experts,
tileTokensDim)`` and must stay <= 65535. Solving that for num_tokens
with the smallest tile the kernel may pick (tileTokensDim=8, the runner
default) gives a bound that is safe regardless of the tactic selected.
Without it, large batches (e.g. Kimi-K3 top_k=16, EP16 profiling with
131072 gathered tokens) overflow the grid and the GEMM launch fails.
"""
MAX_GRID_Y = 65535
MIN_TILE_TOKENS_DIM = 8
max_tokens = (MAX_GRID_Y - global_num_experts) * MIN_TILE_TOKENS_DIM // top_k
return max(1, min(300000, max_tokens))
def _invoke_kernel(
self,
output: torch.Tensor,
x_quant: torch.Tensor,
x_scale: torch.Tensor | None,
topk_ids: torch.Tensor,
topk_weights: torch.Tensor,
w1: torch.Tensor,
w2: torch.Tensor,
activation: MoEActivation,
global_num_experts: int,
local_num_experts: int,
local_expert_offset: int,
topk: int,
) -> None:
from flashinfer import trtllm_fp4_block_scale_routed_moe
packed_tensor = trtllm_moe_pack_topk_ids_weights(topk_ids, topk_weights)
trtllm_fp4_block_scale_routed_moe(
topk_ids=packed_tensor,
routing_bias=None,
hidden_states=x_quant,
hidden_states_scale=x_scale,
gemm1_weights=w1,
gemm1_weights_scale=self.w1_scale,
gemm1_bias=self.w1_bias,
gemm1_alpha=self.gemm1_alpha,
gemm1_beta=self.gemm1_beta,
gemm1_clamp_limit=self.gemm1_clamp_limit,
gemm2_weights=w2,
gemm2_weights_scale=self.w2_scale,
gemm2_bias=self.w2_bias,
output1_scale_scalar=None,
output1_scale_gate_scalar=None,
output2_scale_scalar=None,
num_experts=global_num_experts,
top_k=topk,
n_group=None,
topk_group=None,
intermediate_size=self.intermediate_size_per_partition,
local_expert_offset=local_expert_offset,
local_num_experts=local_num_experts,
routed_scaling_factor=None,
# Modular kernel receives pre-routed tokens, so routing is already
# done. Use Renormalize as a safe default the TRTLLM kernel supports.
routing_method_type=RoutingMethodType.Renormalize,
do_finalize=True,
enable_pdl=True,
activation_type=self._flashinfer_activation_type(activation),
output=output,
tune_max_num_tokens=max(self.max_capture_size, 1),
)
def apply(
self,
output: torch.Tensor,
@@ -335,7 +405,6 @@ class TrtLlmMxfp4ExpertsModular(TrtLlmMxfp4ExpertsBase, mk.FusedMoEExpertsModula
):
topk = topk_ids.size(-1)
local_num_experts = w1.size(0)
intermediate_size = self.intermediate_size_per_partition
local_expert_offset = self.moe_config.ep_rank * local_num_experts
if a1q_scale is not None:
@@ -346,49 +415,27 @@ class TrtLlmMxfp4ExpertsModular(TrtLlmMxfp4ExpertsBase, mk.FusedMoEExpertsModula
x_quant = hidden_states
x_scale = None
# Pack topk ids and weights into format expected by the kernel.
packed_tensor = trtllm_moe_pack_topk_ids_weights(topk_ids, topk_weights)
assert self.w1_scale is not None
assert self.w2_scale is not None
kwargs = {
"topk_ids": packed_tensor,
"routing_bias": None,
"hidden_states": x_quant,
"hidden_states_scale": x_scale,
"gemm1_weights": w1,
"gemm1_weights_scale": self.w1_scale,
"gemm1_bias": self.w1_bias,
"gemm1_alpha": self.gemm1_alpha,
"gemm1_beta": self.gemm1_beta,
"gemm1_clamp_limit": self.gemm1_clamp_limit,
"gemm2_weights": w2,
"gemm2_weights_scale": self.w2_scale,
"gemm2_bias": self.w2_bias,
"output1_scale_scalar": None,
"output1_scale_gate_scalar": None,
"output2_scale_scalar": None,
"num_experts": global_num_experts,
"top_k": topk,
"n_group": None,
"topk_group": None,
"intermediate_size": intermediate_size,
"local_expert_offset": local_expert_offset,
"local_num_experts": local_num_experts,
"routed_scaling_factor": None,
# Modular kernel receives pre-routed tokens, so routing
# is already done. Use Renormalize as a safe default that
# the TRTLLM C++ kernel supports.
"routing_method_type": RoutingMethodType.Renormalize,
"do_finalize": True,
"enable_pdl": True,
"activation_type": self._flashinfer_activation_type(activation),
"output": output,
"tune_max_num_tokens": max(self.max_capture_size, 1),
}
from flashinfer import trtllm_fp4_block_scale_routed_moe
trtllm_fp4_block_scale_routed_moe(**kwargs)
# Chunk tokens so the batched-GEMM grid stays within CUDA limits.
M = x_quant.size(0)
chunk_size = self._max_supported_tokens(topk, global_num_experts)
for start in range(0, M, chunk_size):
end = min(start + chunk_size, M)
self._invoke_kernel(
output[start:end],
x_quant[start:end],
None if x_scale is None else x_scale[start:end],
topk_ids[start:end],
topk_weights[start:end],
w1,
w2,
activation,
global_num_experts,
local_num_experts,
local_expert_offset,
topk,
)
return output
@@ -22,6 +22,7 @@ from vllm.model_executor.layers.fused_moe.utils import (
)
from vllm.model_executor.layers.quantization.utils.flashinfer_utils import (
activation_to_flashinfer_int,
has_flashinfer_situ_activation,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
QuantKey,
@@ -225,15 +226,15 @@ class TrtLlmNvFp4ExpertsBase:
@staticmethod
def _supports_activation(activation: MoEActivation) -> bool:
"""Supports SiLU, RELU^2 non-gated, GELU, clamped SwiGLU-OAI, and the
private SituGLU (SITU) cubin."""
"""Supports SITU only when the installed FlashInfer exposes it."""
if activation == MoEActivation.SITU:
return has_flashinfer_situ_activation()
return activation in [
MoEActivation.SILU,
MoEActivation.RELU2_NO_MUL,
MoEActivation.GELU,
MoEActivation.GELU_TANH,
MoEActivation.SWIGLUOAI_UNINTERLEAVE,
MoEActivation.SITU,
]
@staticmethod
@@ -648,7 +648,7 @@ def mxfp4_round_up_hidden_size_and_intermediate_size(
else:
hidden_size = round_up(hidden_size, 256)
elif backend in TRTLLM_BACKENDS:
intermediate_size = round_up(intermediate_size, 256)
intermediate_size = round_up(intermediate_size, 128)
hidden_size = round_up(hidden_size, 256)
elif backend in (
Mxfp4MoeBackend.FLASHINFER_CUTLASS_MXFP4_BF16,
@@ -102,7 +102,7 @@ class BatchedPrepareAndFinalize(mk.FusedMoEPrepareAndFinalizeModular):
num_local_experts, self.max_num_tokens, hidden_dim
)
b_a1_scale = torch.empty(scale_shape, dtype=torch.float32, device=a1.device)
b_a1_scale = torch.zeros(scale_shape, dtype=torch.float32, device=a1.device)
else:
assert quant_config.a1_scale is None
b_a1_scale = None
@@ -6,11 +6,6 @@ import vllm.envs as envs
from vllm.config import get_current_vllm_config
from vllm.distributed import tensor_model_parallel_all_reduce
from vllm.logger import init_logger
from vllm.model_executor.layers.fused_allreduce_gemma_rms_norm import (
_AR_RESIDUAL_RMS_NORM,
_can_use_flashinfer,
flashinfer_trtllm_fused_allreduce_norm,
)
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.utils.torch_utils import aux_stream, current_stream
@@ -82,25 +77,30 @@ class LatentMoERunner(MoERunner):
)
self._k3_latent_moe_tail_op = op
def _get_zero_residual(self, hidden_states: torch.Tensor) -> torch.Tensor:
def _get_zero_residual(
self,
hidden_states: torch.Tensor,
max_token_num: int,
) -> torch.Tensor:
"""Read-only zero ``residual_in`` for the fused AR+RMSNorm kernel.
flashinfer requires a residual buffer even when there is no residual to
add.
"""
numel = hidden_states.numel()
buf = getattr(self, "_zero_residual", None)
if (
buf is None
or buf.dtype != hidden_states.dtype
or buf.device != hidden_states.device
or buf.numel() < numel
):
if buf is None:
buf = torch.zeros(
numel, dtype=hidden_states.dtype, device=hidden_states.device
max_token_num * hidden_states.shape[-1],
dtype=hidden_states.dtype,
device=hidden_states.device,
)
self._zero_residual = buf
return buf[:numel].view_as(hidden_states)
assert buf.dtype == hidden_states.dtype
assert buf.device == hidden_states.device
assert hidden_states.numel() <= buf.numel()
return buf[: hidden_states.numel()].view_as(hidden_states)
def _use_fused_path(self) -> bool:
# The fused path merges the latent and shared reductions into one
@@ -224,6 +224,12 @@ class LatentMoERunner(MoERunner):
norm: RMSNorm,
) -> tuple[torch.Tensor, torch.Tensor]:
"""All-reduce + add residual + (standard) RMSNorm, fused via flashinfer."""
from vllm.model_executor.layers.fused_allreduce_gemma_rms_norm import (
_AR_RESIDUAL_RMS_NORM,
_can_use_flashinfer,
flashinfer_trtllm_fused_allreduce_norm,
)
if self.moe_config.tp_size == 1:
return norm(hidden_states)
@@ -238,7 +244,7 @@ class LatentMoERunner(MoERunner):
# buffer and the normalized result into norm_out.
flashinfer_trtllm_fused_allreduce_norm(
allreduce_in=hidden_states,
residual=self._get_zero_residual(hidden_states),
residual=self._get_zero_residual(hidden_states, max_token_num),
rms_gamma=norm.weight,
rms_eps=norm.variance_epsilon,
world_size=self.moe_config.tp_size,
@@ -1,32 +1,33 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Callable
import torch
from einops import rearrange
from torch import nn
from torch.nn.parameter import Parameter
from vllm.config import VllmConfig, get_current_vllm_config
from vllm.distributed import (
divide,
)
from vllm.forward_context import ForwardContext, get_forward_context
from vllm.logger import init_logger
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
from vllm.config import VllmConfig
from vllm.distributed import divide, get_tensor_model_parallel_rank
from vllm.forward_context import get_forward_context
from vllm.model_executor.custom_op import PluggableLayer
from vllm.model_executor.layers.mamba.gdn.base import GatedDeltaNetAttention
from vllm.model_executor.model_loader.weight_utils import sharded_weight_loader
from vllm.model_executor.utils import set_weight_attrs
from vllm.third_party.flash_linear_attention.ops.kda import (
FusedRMSNormGated,
chunk_kda_with_fused_gate,
fused_kda_gate,
fused_recurrent_kda,
from vllm.model_executor.model_loader.weight_utils import (
default_weight_loader,
sharded_weight_loader,
)
from vllm.model_executor.parameter import BasevLLMParameter
from vllm.model_executor.utils import set_weight_attrs
from vllm.platforms import current_platform
from vllm.third_party.flash_linear_attention.ops.kda import FusedRMSNormGated
from vllm.transformers_utils.configs.kimi_linear import KimiLinearConfig
from vllm.utils.torch_utils import direct_register_custom_op
from vllm.v1.attention.backends.gdn_attn import GDNAttentionMetadata
from ...linear import (
ColumnParallelLinear,
MergedColumnParallelLinear,
ReplicatedLinear,
RowParallelLinear,
)
@@ -36,49 +37,118 @@ from ..mamba_utils import (
is_conv_state_dim_first,
)
from ..ops.causal_conv1d import causal_conv1d_fn, causal_conv1d_update
from ..ops.gather_initial_states import gather_initial_states
logger = init_logger(__name__)
# Empirical lower bound for the KDA gate to avoid numerical underflow.
_KDA_GATE_LOGBOUND_MIN = -5.0
def kda_attention(
q_proj_states: torch.Tensor,
k_proj_states: torch.Tensor,
v_proj_states: torch.Tensor,
g1: torch.Tensor,
beta: torch.Tensor,
core_attn_out: torch.Tensor,
layer_name: str,
) -> None:
forward_context: ForwardContext = get_forward_context()
self = forward_context.no_compile_layers[layer_name]
self._forward(
q_proj_states=q_proj_states,
k_proj_states=k_proj_states,
v_proj_states=v_proj_states,
g1=g1,
beta=beta,
core_attn_out=core_attn_out,
)
def a_log_weight_loader(
shard_axis: int,
) -> Callable[[torch.Tensor, torch.Tensor], None]:
"""Load KDA A_log stored as either old 4D or current 1D weights."""
def loader(param: torch.Tensor, loaded_weight: torch.Tensor) -> None:
tp_rank = get_tensor_model_parallel_rank()
shard_size = param.data.shape[shard_axis]
start_idx = tp_rank * shard_size
if loaded_weight.dim() == 4:
assert loaded_weight.shape[:2] == (1, 1), (
f"Expected old A_log shape (1, 1, H, 1), got {loaded_weight.shape}"
)
assert loaded_weight.shape[-1] == 1, (
f"Expected old A_log last dim to be 1, got {loaded_weight.shape}"
)
loaded_weight = loaded_weight.view(loaded_weight.shape[2])
loaded_weight = loaded_weight.narrow(shard_axis, start_idx, shard_size)
return default_weight_loader(param, loaded_weight)
return loader
def kda_attention_fake(
q_proj_states: torch.Tensor,
k_proj_states: torch.Tensor,
v_proj_states: torch.Tensor,
g1: torch.Tensor,
beta: torch.Tensor,
core_attn_out: torch.Tensor,
layer_name: str,
) -> None:
return
def _make_fused_conv1d_weight_loader(
dims: list[int],
tp_size: int,
tp_rank: int,
) -> Callable[..., None]:
sharded_dims = [dim // tp_size for dim in dims]
def weight_loader(
param: torch.Tensor,
loaded_weight: torch.Tensor,
loaded_shard_id: int,
) -> None:
if loaded_weight.dim() == 2:
loaded_weight = loaded_weight.unsqueeze(1)
shard_size = sharded_dims[loaded_shard_id]
source_start = tp_rank * shard_size
target_start = sum(sharded_dims[:loaded_shard_id])
loaded_shard = loaded_weight[source_start : source_start + shard_size]
param.data[target_start : target_start + shard_size].copy_(loaded_shard)
return weight_loader
direct_register_custom_op(
op_name="kda_attention",
op_func=kda_attention,
mutates_args=["core_attn_out"],
fake_impl=kda_attention_fake,
)
class _KimiGDNMergedColumnParallelLinear(MergedColumnParallelLinear):
"""Merged projection with one output replicated across TP ranks.
The replicated shard is represented as ``size * tp_size`` so the merged
parameter reserves ``size`` local rows on every rank. Loading that shard
from rank zero then gives every rank the complete checkpoint weight.
"""
def __init__(
self,
input_size: int,
output_sizes: list[int],
replicated_shard_id: int,
tp_size: int,
**kwargs,
) -> None:
self.replicated_shard_id = replicated_shard_id
output_sizes = output_sizes.copy()
output_sizes[replicated_shard_id] *= tp_size
super().__init__(input_size, output_sizes, **kwargs)
def weight_loader(
self,
param: Parameter,
loaded_weight: torch.Tensor,
loaded_shard_id: tuple[int, ...] | int | None = None,
) -> None:
tp_rank = self.tp_rank
param_tp_rank = getattr(param, "tp_rank", None)
if loaded_shard_id == self.replicated_shard_id:
self.tp_rank = 0
if param_tp_rank is not None:
param.tp_rank = 0
try:
super().weight_loader(param, loaded_weight, loaded_shard_id)
finally:
self.tp_rank = tp_rank
if param_tp_rank is not None:
param.tp_rank = param_tp_rank
def weight_loader_v2(
self,
param: BasevLLMParameter,
loaded_weight: torch.Tensor,
loaded_shard_id: tuple[int, ...] | int | None = None,
) -> None:
tp_rank = self.tp_rank
param_tp_rank = getattr(param, "tp_rank", None)
if loaded_shard_id == self.replicated_shard_id:
self.tp_rank = 0
if param_tp_rank is not None:
param.tp_rank = 0
try:
super().weight_loader_v2(param, loaded_weight, loaded_shard_id)
finally:
self.tp_rank = tp_rank
if param_tp_rank is not None:
param.tp_rank = param_tp_rank
@PluggableLayer.register("kimi_gated_delta_net_attention")
@@ -96,7 +166,11 @@ class KimiGatedDeltaNetAttention(GatedDeltaNetAttention):
self,
) -> tuple[tuple[int, ...], tuple[int, ...]]:
return MambaStateShapeCalculator.kda_state_shape(
self.tp_size, self.num_heads, self.head_dim, conv_kernel_size=self.conv_size
self.tp_size,
self.num_heads,
self.head_dim,
conv_kernel_size=self.conv_size,
num_spec=self.num_spec,
)
def __init__(
@@ -114,122 +188,142 @@ class KimiGatedDeltaNetAttention(GatedDeltaNetAttention):
assert self.num_heads % self.tp_size == 0
self.local_num_heads = divide(self.num_heads, self.tp_size)
projection_size = self.head_dim * self.num_heads
self.projection_size = self.head_dim * self.num_heads
self.local_projection_size = divide(self.projection_size, self.tp_size)
self.conv_size = kda_config["short_conv_kernel_size"]
self.use_full_rank_gate = kda_config.get("use_full_rank_gate", False)
self.q_proj = ColumnParallelLinear(
if self.use_full_rank_gate:
# Keep f_a before the narrow beta shard, then pad each TP-local row
# to select the aligned BF16 GEMM path. The padding also avoids an
# Inductor correctness issue seen with the row-strided G view.
qkvg_output_sizes = [self.projection_size] * 4
in_proj_output_sizes = qkvg_output_sizes + [
self.head_dim,
self.num_heads,
]
local_output_size = (
4 * self.local_projection_size + self.head_dim + self.local_num_heads
)
self.in_proj_padding = -local_output_size % 16
if self.in_proj_padding:
in_proj_output_sizes.append(self.in_proj_padding * self.tp_size)
else:
in_proj_output_sizes = [self.projection_size] * 3 + [
self.num_heads,
self.head_dim,
]
self.in_proj_padding = 0
self.in_proj_qkvgfab = _KimiGDNMergedColumnParallelLinear(
self.hidden_size,
projection_size,
in_proj_output_sizes,
replicated_shard_id=4,
tp_size=self.tp_size,
bias=False,
quant_config=self.quant_config,
prefix=f"{prefix}.q_proj",
)
self.k_proj = ColumnParallelLinear(
self.hidden_size,
projection_size,
bias=False,
quant_config=self.quant_config,
prefix=f"{prefix}.k_proj",
)
self.v_proj = ColumnParallelLinear(
self.hidden_size,
projection_size,
bias=False,
quant_config=self.quant_config,
prefix=f"{prefix}.v_proj",
)
self.f_a_proj = ReplicatedLinear(
self.hidden_size,
self.head_dim,
bias=False,
quant_config=self.quant_config,
prefix=f"{prefix}.f_a_proj",
prefix=f"{prefix}.in_proj_qkvgfab",
)
if self.in_proj_padding:
self.in_proj_qkvgfab.weight.data[-self.in_proj_padding :].zero_()
self.f_b_proj = ColumnParallelLinear(
self.head_dim,
projection_size,
self.projection_size,
bias=False,
quant_config=self.quant_config,
prefix=f"{prefix}.f_b_proj",
)
self.dt_bias = nn.Parameter(
torch.empty(divide(projection_size, self.tp_size), dtype=torch.float32)
torch.empty(self.local_projection_size, dtype=torch.float32)
)
set_weight_attrs(self.dt_bias, {"weight_loader": sharded_weight_loader(0)})
self.b_proj = ColumnParallelLinear(
self.hidden_size,
self.num_heads,
bias=False,
quant_config=self.quant_config,
prefix=f"{prefix}.b_proj",
)
self.q_conv1d = ColumnParallelLinear(
# One packed parameter and cache let decode run a single conv update.
# Prefill slices them back into Q/K/V to obtain dense outputs cheaply.
self.conv1d = ColumnParallelLinear(
input_size=self.conv_size,
output_size=projection_size,
output_size=3 * self.projection_size,
bias=False,
params_dtype=torch.float32,
prefix=f"{prefix}.q_conv1d",
prefix=f"{prefix}.conv1d",
)
self.k_conv1d = ColumnParallelLinear(
input_size=self.conv_size,
output_size=projection_size,
bias=False,
params_dtype=torch.float32,
prefix=f"{prefix}.k_conv1d",
self.conv1d.weight.data = self.conv1d.weight.data.unsqueeze(1)
delattr(self.conv1d.weight, "weight_loader")
set_weight_attrs(
self.conv1d.weight,
{
"weight_loader": _make_fused_conv1d_weight_loader(
[self.projection_size] * 3,
self.tp_size,
self.tp_rank,
)
},
)
self.v_conv1d = ColumnParallelLinear(
input_size=self.conv_size,
output_size=projection_size,
bias=False,
params_dtype=torch.float32,
prefix=f"{prefix}.v_conv1d",
)
# unsqueeze to fit conv1d weights shape into the linear weights shape.
# Can't do this in `weight_loader` since it already exists in
# `ColumnParallelLinear` and `set_weight_attrs`
# doesn't allow to override it
self.q_conv1d.weight.data = self.q_conv1d.weight.data.unsqueeze(1)
self.k_conv1d.weight.data = self.k_conv1d.weight.data.unsqueeze(1)
self.v_conv1d.weight.data = self.v_conv1d.weight.data.unsqueeze(1)
self.A_log = nn.Parameter(
torch.empty(1, 1, self.local_num_heads, 1, dtype=torch.float32)
torch.empty(self.local_num_heads, dtype=torch.float32)
)
set_weight_attrs(self.A_log, {"weight_loader": sharded_weight_loader(2)})
set_weight_attrs(self.A_log, {"weight_loader": a_log_weight_loader(0)})
self.g_a_proj = ReplicatedLinear(
self.hidden_size,
self.head_dim,
bias=False,
quant_config=self.quant_config,
prefix=f"{prefix}.g_a_proj",
self.gate_lower_bound: float | None = kda_config.get("gate_lower_bound", None)
if self.gate_lower_bound is not None:
assert _KDA_GATE_LOGBOUND_MIN <= self.gate_lower_bound < 0, (
"KDA gate lower bound must be in "
f"[{_KDA_GATE_LOGBOUND_MIN}, 0). "
f"Got {self.gate_lower_bound}."
)
self.use_safe_gate = self.gate_lower_bound is not None
additional_config = vllm_config.additional_config
backend = (
additional_config.get("kda_prefill_backend", "auto")
if isinstance(additional_config, dict)
else "auto"
)
self.g_b_proj = ColumnParallelLinear(
self.head_dim,
projection_size,
bias=False,
quant_config=self.quant_config,
prefix=f"{prefix}.g_b_proj",
backend = "triton" if backend == "auto" else backend
assert backend == "triton", (
"The shared Kimi GDN layer only supports the Triton KDA "
f"prefill backend, got {backend!r}."
)
if not self.use_full_rank_gate:
self.g_a_proj = ReplicatedLinear(
self.hidden_size,
self.head_dim,
bias=False,
quant_config=self.quant_config,
prefix=f"{prefix}.g_a_proj",
)
self.g_b_proj = ColumnParallelLinear(
self.head_dim,
self.projection_size,
bias=False,
quant_config=self.quant_config,
prefix=f"{prefix}.g_b_proj",
)
self.o_norm = FusedRMSNormGated(self.head_dim, activation="sigmoid")
self.o_proj = RowParallelLinear(
projection_size,
self.projection_size,
self.hidden_size,
bias=False,
quant_config=self.quant_config,
prefix=f"{prefix}.o_proj",
)
compilation_config = get_current_vllm_config().compilation_config
compilation_config = vllm_config.compilation_config
if prefix in compilation_config.static_forward_context:
raise ValueError(f"Duplicate layer name: {prefix}")
compilation_config.static_forward_context[prefix] = self
def rearrange_mixed_qkv(
self, mixed_qkv: torch.Tensor
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
seq_len = mixed_qkv.shape[0]
qkv = mixed_qkv.view(seq_len, 3, self.local_num_heads, self.head_dim)
# Materialize all three row-strided inputs with one token-major to
# QKV-major permutation. Each unbound tensor is then contiguous.
qkv = qkv.permute(1, 0, 2, 3).contiguous().unsqueeze(1)
return qkv.unbind(0)
def forward(
self,
hidden_states: torch.Tensor,
@@ -237,42 +331,57 @@ class KimiGatedDeltaNetAttention(GatedDeltaNetAttention):
output: torch.Tensor,
) -> None:
num_tokens = hidden_states.size(0)
q = self.q_proj(hidden_states)[0]
k = self.k_proj(hidden_states)[0]
v = self.v_proj(hidden_states)[0]
projected_qkvgfab = self.in_proj_qkvgfab(hidden_states)[0]
if self.use_full_rank_gate:
split_sizes = [
3 * self.local_projection_size,
self.local_projection_size,
self.head_dim,
self.local_num_heads,
]
if self.in_proj_padding:
split_sizes.append(self.in_proj_padding)
projected = projected_qkvgfab.split(split_sizes, dim=-1)
mixed_qkv, g_proj_states, f_a, beta = projected[:4]
else:
mixed_qkv, beta, f_a = projected_qkvgfab.split(
[
3 * self.local_projection_size,
self.local_num_heads,
self.head_dim,
],
dim=-1,
)
g_proj_states = self.g_b_proj(self.g_a_proj(hidden_states)[0])[0]
beta = self.b_proj(hidden_states)[0].float().sigmoid()
g1 = self.f_b_proj(self.f_a_proj(hidden_states)[0])[0]
g1 = self.f_b_proj(f_a)[0]
beta = beta.unsqueeze(0)
g1 = rearrange(g1, "n (h d) -> 1 n h d", d=self.head_dim)
g_proj_states = self.g_b_proj(self.g_a_proj(hidden_states)[0])[0]
g2 = rearrange(g_proj_states, "... (h d) -> ... h d", d=self.head_dim)
core_attn_out = torch.zeros(
core_attn_out = torch.empty(
(1, num_tokens, self.local_num_heads, self.head_dim),
dtype=hidden_states.dtype,
device=hidden_states.device,
)
torch.ops.vllm.kda_attention(
q,
k,
v,
g1,
beta,
core_attn_out,
self.prefix,
self._forward(
mixed_qkv=mixed_qkv,
g1=g1,
g2=g2,
beta=beta,
core_attn_out=core_attn_out,
)
core_attn_out = self.o_norm(core_attn_out, g2)
core_attn_out = rearrange(core_attn_out, "1 n h d -> n (h d)")
output[:] = self.o_proj(core_attn_out)[0]
@eager_break_during_capture
def _forward(
self,
q_proj_states: torch.Tensor,
k_proj_states: torch.Tensor,
v_proj_states: torch.Tensor,
mixed_qkv: torch.Tensor,
g1: torch.Tensor,
g2: torch.Tensor,
beta: torch.Tensor,
core_attn_out: torch.Tensor,
) -> None:
@@ -280,165 +389,247 @@ class KimiGatedDeltaNetAttention(GatedDeltaNetAttention):
attn_metadata_raw = forward_context.attn_metadata
if attn_metadata_raw is None:
# # V1 profile run
return
# Vendor-specific KDA kernels: AMD/ROCm and NVIDIA keep their own copies
# under kimi_k3/{amd,nvidia}/ops so each can diverge independently.
if current_platform.is_rocm():
from vllm.models.kimi_k3.amd.ops.third_party.kda import (
chunk_kda_with_fused_gate,
fused_recurrent_kda,
fused_recurrent_kda_packed_decode,
)
else:
from vllm.models.kimi_k3.nvidia.ops.third_party.kda import (
chunk_kda_with_fused_gate,
fused_recurrent_kda,
fused_recurrent_kda_packed_decode,
)
assert isinstance(attn_metadata_raw, dict)
attn_metadata_narrowed = attn_metadata_raw[self.prefix]
assert isinstance(attn_metadata_narrowed, GDNAttentionMetadata)
has_initial_state = attn_metadata_narrowed.has_initial_state
non_spec_query_start_loc = attn_metadata_narrowed.non_spec_query_start_loc
non_spec_state_indices_tensor = (
attn_metadata_narrowed.non_spec_state_indices_tensor
) # noqa: E501
num_actual_tokens = attn_metadata_narrowed.num_actual_tokens
constant_caches = self.kv_cache
q_proj_states = q_proj_states[:num_actual_tokens]
k_proj_states = k_proj_states[:num_actual_tokens]
v_proj_states = v_proj_states[:num_actual_tokens]
m = attn_metadata_narrowed
has_initial_state = m.has_initial_state
non_spec_query_start_loc = m.non_spec_query_start_loc
non_spec_state_indices_tensor = m.non_spec_state_indices_tensor
spec_sequence_masks = m.spec_sequence_masks
spec_token_indx = m.spec_token_indx
non_spec_token_indx = m.non_spec_token_indx
spec_state_indices_tensor = m.spec_state_indices_tensor
spec_query_start_loc = m.spec_query_start_loc
num_accepted_tokens = m.num_accepted_tokens
num_actual_tokens = m.num_actual_tokens
mixed_qkv = mixed_qkv[:num_actual_tokens]
g1 = g1[:, :num_actual_tokens]
beta = beta[:, :num_actual_tokens]
(conv_state, recurrent_state) = constant_caches
constant_caches = self.kv_cache
conv_state, recurrent_state = constant_caches
# conv_state must be (..., dim, width-1) for the conv kernels.
# DS layout stores it that way directly; SD layout needs a transpose.
if not is_conv_state_dim_first():
conv_state = conv_state.transpose(-1, -2)
conv_state_q, conv_state_k, conv_state_v = conv_state.chunk(3, dim=-2)
conv_weights = self.conv1d.weight.view(
self.conv1d.weight.size(0), self.conv1d.weight.size(2)
)
q_conv_weight, k_conv_weight, v_conv_weight = conv_weights.split(
self.local_projection_size, dim=0
)
q_conv_state, k_conv_state, v_conv_state = conv_state.split(
self.local_projection_size, dim=-2
)
q_conv_weights = self.q_conv1d.weight.view(
self.q_conv1d.weight.size(0), self.q_conv1d.weight.size(2)
)
k_conv_weights = self.k_conv1d.weight.view(
self.k_conv1d.weight.size(0), self.k_conv1d.weight.size(2)
)
v_conv_weights = self.v_conv1d.weight.view(
self.v_conv1d.weight.size(0), self.v_conv1d.weight.size(2)
)
if attn_metadata_narrowed.num_prefills > 0:
q_proj_states = q_proj_states.transpose(0, 1)
k_proj_states = k_proj_states.transpose(0, 1)
v_proj_states = v_proj_states.transpose(0, 1)
q = causal_conv1d_fn(
q_proj_states,
q_conv_weights,
self.q_conv1d.bias,
activation="silu",
conv_states=conv_state_q,
has_initial_state=has_initial_state,
cache_indices=non_spec_state_indices_tensor,
query_start_loc=non_spec_query_start_loc,
metadata=attn_metadata_narrowed,
).transpose(0, 1)
k = causal_conv1d_fn(
k_proj_states,
k_conv_weights,
self.k_conv1d.bias,
activation="silu",
conv_states=conv_state_k,
has_initial_state=has_initial_state,
cache_indices=non_spec_state_indices_tensor,
query_start_loc=non_spec_query_start_loc,
metadata=attn_metadata_narrowed,
).transpose(0, 1)
v = causal_conv1d_fn(
v_proj_states,
v_conv_weights,
self.v_conv1d.bias,
activation="silu",
conv_states=conv_state_v,
has_initial_state=has_initial_state,
cache_indices=non_spec_state_indices_tensor,
query_start_loc=non_spec_query_start_loc,
metadata=attn_metadata_narrowed,
).transpose(0, 1)
# Split tokens into the multi-query spec-decode part and the remaining
# (prefill / plain decode) part.
if spec_sequence_masks is not None:
if m.num_prefills == 0 and m.num_decodes == 0:
mixed_qkv_spec = mixed_qkv
g1_spec, beta_spec = g1, beta
mixed_qkv_ns = g1_ns = beta_ns = None
else:
mixed_qkv_spec = mixed_qkv.index_select(0, spec_token_indx)
g1_spec = g1.index_select(1, spec_token_indx)
beta_spec = beta.index_select(1, spec_token_indx)
mixed_qkv_ns = mixed_qkv.index_select(0, non_spec_token_indx)
g1_ns = g1.index_select(1, non_spec_token_indx)
beta_ns = beta.index_select(1, non_spec_token_indx)
else:
assert non_spec_state_indices_tensor is not None
decode_conv_indices = non_spec_state_indices_tensor[
: attn_metadata_narrowed.num_actual_tokens
]
q = causal_conv1d_update(
q_proj_states,
conv_state_q,
q_conv_weights,
self.q_conv1d.bias,
activation="silu",
conv_state_indices=decode_conv_indices,
validate_data=True,
)
k = causal_conv1d_update(
k_proj_states,
conv_state_k,
k_conv_weights,
self.k_conv1d.bias,
activation="silu",
conv_state_indices=decode_conv_indices,
validate_data=True,
)
v = causal_conv1d_update(
v_proj_states,
conv_state_v,
v_conv_weights,
self.v_conv1d.bias,
activation="silu",
conv_state_indices=decode_conv_indices,
validate_data=True,
)
mixed_qkv_spec = g1_spec = beta_spec = None
mixed_qkv_ns, g1_ns, beta_ns = mixed_qkv, g1, beta
q, k, v = map(
lambda x: rearrange(x, "n (h d) -> 1 n h d", d=self.head_dim), (q, k, v)
)
# ---------- spec-decode multi-query path ----------
core_attn_out_spec = None
if spec_sequence_masks is not None:
assert spec_state_indices_tensor is not None
assert spec_query_start_loc is not None
spec_conv_indices = spec_state_indices_tensor[:, 0][: m.num_spec_decodes]
spec_max_query_len = spec_state_indices_tensor.size(-1)
if attn_metadata_narrowed.num_prefills > 0:
assert non_spec_state_indices_tensor is not None
assert has_initial_state is not None
zero_idx = non_spec_state_indices_tensor[~has_initial_state]
recurrent_state[zero_idx] = 0
initial_state = recurrent_state[non_spec_state_indices_tensor].contiguous()
(
core_attn_out_non_spec,
last_recurrent_state,
) = chunk_kda_with_fused_gate(
q=q,
k=k,
v=v,
raw_g=g1,
beta=beta,
# Sibling beta and, for full-rank gates, output-gate views remain
# live, so write the convolution output separately.
spec_conv_out = torch.empty(
mixed_qkv_spec.shape,
dtype=mixed_qkv_spec.dtype,
device=mixed_qkv_spec.device,
)
mixed_qkv_spec = causal_conv1d_update(
mixed_qkv_spec,
conv_state,
conv_weights,
self.conv1d.bias,
activation="silu",
conv_state_indices=spec_conv_indices,
num_accepted_tokens=num_accepted_tokens,
query_start_loc=spec_query_start_loc,
max_query_len=spec_max_query_len,
validate_data=False,
out=spec_conv_out,
)
q_spec, k_spec, v_spec = (
rearrange(x, "n (h d) -> 1 n h d", d=self.head_dim)
for x in mixed_qkv_spec.split(self.local_projection_size, dim=-1)
)
spec_cu_seqlens = spec_query_start_loc[: m.num_spec_decodes + 1]
# Spec-only batches write directly into core_attn_out.
spec_out = (
core_attn_out[:, : q_spec.shape[1]]
if m.num_prefills == 0 and m.num_decodes == 0
else None
)
core_attn_out_spec, _ = fused_recurrent_kda(
q=q_spec,
k=k_spec,
v=v_spec,
raw_g=g1_spec,
raw_beta=beta_spec,
A_log=self.A_log,
g_bias=self.dt_bias,
initial_state=initial_state,
output_final_state=True,
use_qk_l2norm_in_kernel=True,
cu_seqlens=non_spec_query_start_loc,
)
# Init cache
recurrent_state[non_spec_state_indices_tensor] = last_recurrent_state
else:
assert non_spec_query_start_loc is not None
g1 = fused_kda_gate(
rearrange(g1, "1 n h d -> n (h d)"),
self.A_log,
self.head_dim,
g_bias=self.dt_bias,
).unsqueeze(0)
(
core_attn_out_non_spec,
last_recurrent_state,
) = fused_recurrent_kda(
q=q,
k=k,
v=v,
g=g1,
beta=beta,
dt_bias=self.dt_bias,
lower_bound=self.gate_lower_bound,
initial_state=recurrent_state,
use_qk_l2norm_in_kernel=True,
cu_seqlens=non_spec_query_start_loc[
: attn_metadata_narrowed.num_decodes + 1
],
ssm_state_indices=non_spec_state_indices_tensor,
cu_seqlens=spec_cu_seqlens,
ssm_state_indices=spec_state_indices_tensor,
num_accepted_tokens=num_accepted_tokens,
out=spec_out,
)
core_attn_out[0, :num_actual_tokens] = core_attn_out_non_spec[
0, :num_actual_tokens
]
# ---------- non-spec path (prefill or plain decode) ----------
core_attn_out_non_spec = None
if mixed_qkv_ns is not None:
assert g1_ns is not None and beta_ns is not None
if m.num_prefills > 0:
q_ns, k_ns, v_ns = mixed_qkv_ns.split(
self.local_projection_size, dim=-1
)
# Packed prefill conv would require copying V solely to make
# it dense for KDA. Separate calls accept the strided inputs
# and produce dense Q/K/V without that extra traffic.
# TODO: Use packed conv once every KDA prefill backend accepts
# row-strided Q/K/V directly.
def _prefill_conv(
x: torch.Tensor,
state: torch.Tensor,
weight: torch.Tensor,
) -> torch.Tensor:
return causal_conv1d_fn(
x.transpose(0, 1),
weight,
None,
activation="silu",
conv_states=state,
has_initial_state=has_initial_state,
cache_indices=non_spec_state_indices_tensor,
query_start_loc=non_spec_query_start_loc,
metadata=m,
).transpose(0, 1)
q_ns = _prefill_conv(q_ns, q_conv_state, q_conv_weight)
k_ns = _prefill_conv(k_ns, k_conv_state, k_conv_weight)
v_ns = _prefill_conv(v_ns, v_conv_state, v_conv_weight)
q_ns, k_ns, v_ns = (
rearrange(x, "n (h d) -> 1 n h d", d=self.head_dim)
for x in (q_ns, k_ns, v_ns)
)
assert non_spec_state_indices_tensor is not None
assert has_initial_state is not None
initial_state = gather_initial_states(
recurrent_state,
non_spec_state_indices_tensor,
has_initial_state,
)
(
core_attn_out_non_spec,
last_recurrent_state,
) = chunk_kda_with_fused_gate(
q=q_ns,
k=k_ns,
v=v_ns,
raw_g=g1_ns,
raw_beta=beta_ns,
A_log=self.A_log,
g_bias=self.dt_bias,
lower_bound=self.gate_lower_bound,
initial_state=initial_state,
output_final_state=True,
use_qk_l2norm_in_kernel=True,
cu_seqlens=non_spec_query_start_loc,
)
# Init cache
recurrent_state[non_spec_state_indices_tensor] = last_recurrent_state
else:
# pure-decode non-spec batch
assert non_spec_state_indices_tensor is not None
decode_conv_indices = non_spec_state_indices_tensor[
: mixed_qkv_ns.size(0)
]
# Sibling beta and, for full-rank gates, output-gate views
# remain live, so write the conv output separately.
packed_conv_out = torch.empty(
mixed_qkv_ns.shape,
dtype=mixed_qkv_ns.dtype,
device=mixed_qkv_ns.device,
)
mixed_qkv_ns = causal_conv1d_update(
mixed_qkv_ns,
conv_state,
conv_weights,
self.conv1d.bias,
activation="silu",
conv_state_indices=decode_conv_indices,
validate_data=True,
out=packed_conv_out,
)
core_attn_out_non_spec, _ = fused_recurrent_kda_packed_decode(
mixed_qkv=mixed_qkv_ns,
raw_g=g1_ns,
raw_beta=beta_ns,
A_log=self.A_log,
dt_bias=self.dt_bias,
lower_bound=self.gate_lower_bound,
initial_state=recurrent_state,
state_indices=decode_conv_indices,
)
# ---------- merge spec and non-spec outputs ----------
if core_attn_out_spec is not None and core_attn_out_non_spec is not None:
# Mixed batches require indexed placement in the original order.
merged = torch.empty(
(1, num_actual_tokens, *core_attn_out_spec.shape[2:]),
dtype=core_attn_out_spec.dtype,
device=core_attn_out_spec.device,
)
merged.index_copy_(1, spec_token_indx, core_attn_out_spec)
merged.index_copy_(1, non_spec_token_indx, core_attn_out_non_spec)
core_attn_out[0, :num_actual_tokens] = merged[0, :num_actual_tokens]
elif core_attn_out_non_spec is not None:
core_attn_out[0, :num_actual_tokens] = core_attn_out_non_spec[
0, :num_actual_tokens
]
else:
assert core_attn_out_spec is not None
core_attn_out.copy_(self.o_norm(core_attn_out, g2))
@@ -288,7 +288,7 @@ class MambaStateShapeCalculator:
conv_dim = proj_size + 2 * proj_k_size
conv_state_shape = cls._orient_conv_shape(
divide(conv_dim, tp_world_size), conv_kernel_size - 1
divide(conv_dim, tp_world_size), conv_kernel_size - 1 + num_spec
)
recurrent_state_shape = (divide(num_heads, tp_world_size), head_dim, head_dim)
return (conv_state_shape, recurrent_state_shape)
@@ -8,11 +8,12 @@
import numpy as np
import torch
from vllm.platforms import current_platform
from vllm.triton_utils import tl, triton
from vllm.v1.attention.backends.utils import NULL_BLOCK_ID, PAD_SLOT_ID
@triton.jit()
@triton.jit(do_not_specialize_on_alignment=["num_cache_lines"])
def _causal_conv1d_fwd_kernel( # continuous batching
# Pointers to matrices
x_ptr, # (dim, cu_seqlen) holding `batch` of actual sequences + padded sequences
@@ -33,7 +34,7 @@ def _causal_conv1d_fwd_kernel( # continuous batching
o_ptr, # (dim, seqlen) - actually pointing to x_ptr
# Matrix dimensions
dim: tl.constexpr,
num_cache_lines: tl.constexpr, # added to support vLLM larger cache lines
num_cache_lines, # added to support vLLM larger cache lines
# Strides
stride_x_dim: tl.constexpr, # stride to get to next feature-value,
stride_x_token: tl.int64, # stride to get to next token (same feature-index, same sequence-index)
@@ -58,6 +59,7 @@ def _causal_conv1d_fwd_kernel( # continuous batching
NP2_STATELEN: tl.constexpr,
BLOCK_M: tl.constexpr,
BLOCK_N: tl.constexpr,
launch_pdl: tl.constexpr,
):
conv_states_ptr = initial_states_ptr
conv_state_indices_ptr = cache_indices_ptr
@@ -68,6 +70,9 @@ def _causal_conv1d_fwd_kernel( # continuous batching
KERNEL_WIDTH - 1
) # can be passed via argument if it's not the same as this value
if launch_pdl:
tl.extra.cuda.gdc_wait()
# one program handles one chunk in a single sequence
# rather than mixing sequences - to make updating initial_states across sequences efficiently
@@ -79,6 +84,8 @@ def _causal_conv1d_fwd_kernel( # continuous batching
idx_feats = tl.program_id(1) * BLOCK_N + tl.arange(0, BLOCK_N)
if idx_seq == pad_slot_id:
if launch_pdl:
tl.extra.cuda.gdc_launch_dependents()
return
sequence_start_index = tl.load(query_start_loc_ptr + idx_seq)
@@ -136,6 +143,8 @@ def _causal_conv1d_fwd_kernel( # continuous batching
if HAS_NULL_BLOCK: # noqa
if conv_states_input_coord == null_block_id:
# not processing as this is a null block (padding)
if launch_pdl:
tl.extra.cuda.gdc_launch_dependents()
return
conv_states_base = (
conv_states_ptr
@@ -408,6 +417,10 @@ def _causal_conv1d_fwd_kernel( # continuous batching
w_ptrs = w_base + (3 * stride_w_width) # [BLOCK_N] tensor
w_col3 = tl.load(w_ptrs, mask_w, other=0.0)
mask_x_1d = idx_feats < dim
if launch_pdl:
tl.extra.cuda.gdc_launch_dependents()
for idx_token in range(segment_len):
acc = acc_preload
@@ -741,11 +754,12 @@ def causal_conv1d_fn(
BLOCK_M=BLOCK_M,
BLOCK_N=256,
num_stages=2,
launch_pdl=current_platform.is_arch_support_pdl(),
)
return out.to(original_x_dtype)
@triton.jit()
@triton.jit(do_not_specialize_on_alignment=["num_cache_lines"])
def _causal_conv1d_update_kernel(
# Pointers to matrices
x_ptr, # (batch, dim, seqlen)
@@ -763,7 +777,7 @@ def _causal_conv1d_update_kernel(
dim: tl.constexpr,
seqlen: tl.constexpr,
state_len: tl.constexpr,
num_cache_lines: tl.constexpr, # added to support vLLM larger cache lines
num_cache_lines, # added to support vLLM larger cache lines
# Strides
stride_x_seq: tl.constexpr,
stride_x_dim: tl.constexpr,
@@ -789,10 +803,16 @@ def _causal_conv1d_update_kernel(
NP2_STATELEN: tl.constexpr,
HAS_NULL_BLOCK: tl.constexpr,
BLOCK_N: tl.constexpr,
launch_pdl: tl.constexpr,
):
if launch_pdl:
tl.extra.cuda.gdc_wait()
# ruff: noqa: E501
idx_seq = tl.program_id(0)
if idx_seq >= batch:
if launch_pdl:
tl.extra.cuda.gdc_launch_dependents()
return
# [BLOCK_N,] elements along the feature-dimension (channel)
@@ -814,6 +834,8 @@ def _causal_conv1d_update_kernel(
if HAS_NULL_BLOCK: # noqa
if conv_states_input_coord == null_block_id:
# not processing as this is not the actual sequence
if launch_pdl:
tl.extra.cuda.gdc_launch_dependents()
return
if IS_VARLEN:
@@ -831,6 +853,8 @@ def _causal_conv1d_update_kernel(
o_offset = idx_seq * stride_o_seq
if query_start_index == query_end_index:
if launch_pdl:
tl.extra.cuda.gdc_launch_dependents()
return
if IS_SPEC_DECODING:
@@ -969,6 +993,9 @@ def _causal_conv1d_update_kernel(
mask_x_1d = idx_feats < dim
# STEP 5: compute each token
if launch_pdl:
tl.extra.cuda.gdc_launch_dependents()
for idx_token in tl.range(seqlen):
acc = acc_preload
@@ -1245,14 +1272,13 @@ def causal_conv1d_update(
NP2_STATELEN=np2_statelen,
HAS_NULL_BLOCK=null_block_id is not None,
BLOCK_N=256,
launch_pdl=current_platform.is_arch_support_pdl(),
)
if unsqueeze:
out = out.squeeze(-1)
return out.to(original_x_dtype)
from vllm.platforms import current_platform # noqa: E402
if current_platform.is_cpu():
from vllm.model_executor.layers.mamba.ops.cpu.causal_conv1d import (
causal_conv1d_fn_cpu,
+7
View File
@@ -28,6 +28,7 @@ class MLAModules:
is_sparse: bool
topk_indices_buffer: torch.Tensor | None
indexer_rotary_emb: torch.nn.Module | None = None
g_proj: torch.nn.Module | None = None
# --8<-- [start:multi_head_latent_attention]
@@ -66,6 +67,7 @@ class MultiHeadLatentAttentionWrapper(PluggableLayer):
quant_config: QuantizationConfig | None = None,
prefix: str = "",
skip_topk: bool = False,
non_causal_multi_token_decode: bool = False,
allow_short_prefill_indexer_scoring_skip: bool = False,
) -> None:
super().__init__()
@@ -89,6 +91,7 @@ class MultiHeadLatentAttentionWrapper(PluggableLayer):
self.indexer = mla_modules.indexer
self.indexer_rope_emb = mla_modules.indexer_rotary_emb
self.is_sparse = mla_modules.is_sparse
self.g_proj = mla_modules.g_proj
# Whether to skip top-k token selection computation in this layer.
# When True, the indexer will not be called, and the layer will reuse
@@ -120,6 +123,7 @@ class MultiHeadLatentAttentionWrapper(PluggableLayer):
use_sparse=self.is_sparse,
indexer=self.indexer,
topk_indices_buffer=mla_modules.topk_indices_buffer,
non_causal_multi_token_decode=non_causal_multi_token_decode,
)
indexer_op = getattr(self.indexer, "indexer_op", None)
if indexer_op is not None and hasattr(
@@ -216,4 +220,7 @@ class MultiHeadLatentAttentionWrapper(PluggableLayer):
q_dcp_replicated=q_dcp_replicated,
)
if self.g_proj is not None:
attn_out = attn_out * self.g_proj(hidden_states)[0].sigmoid()
return self.o_proj(attn_out)[0]
@@ -3,6 +3,7 @@
import torch
import vllm.envs as envs
from vllm.logger import init_logger
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.fused_moe import (
@@ -17,6 +18,7 @@ from vllm.model_executor.layers.fused_moe import modular_kernel as mk
from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import (
TRITON_BACKENDS,
Mxfp4MoeBackend,
backend_to_kernel_cls,
convert_gpt_oss_weight_to_mxfp4_moe_kernel_format,
convert_weight_to_mxfp4_moe_kernel_format,
make_mxfp4_moe_kernel,
@@ -68,8 +70,11 @@ class Mxfp4Config(QuantizationConfig):
def get_config_filenames(cls) -> list[str]:
return []
# TODO (zyongye) This is only temporaty fallback.
# We should have `Mxfp4MoEMethod` after this migration is complete.
def _make_moe_method(self, moe: FusedMoEConfig) -> FusedMoEMethodBase:
"""MoE method for RoutedExperts. Subclasses override to pick a
checkpoint-specific kernel family."""
return Mxfp4MoEMethod(moe)
def get_quant_method(
self, layer: torch.nn.Module, prefix: str
) -> "QuantizeMethodBase | None":
@@ -86,7 +91,7 @@ class Mxfp4Config(QuantizationConfig):
)
return UnquantizedLinearMethod()
elif isinstance(layer, RoutedExperts):
return GptOssMxfp4MoEMethod(layer.moe_config)
return self._make_moe_method(layer.moe_config)
elif isinstance(layer, Attention):
logger.debug_once(
"MXFP4 attention layer is not implemented. "
@@ -131,6 +136,9 @@ class GptOssMxfp4Config(Mxfp4Config):
return None
return "gpt_oss_mxfp4"
def _make_moe_method(self, moe: FusedMoEConfig) -> FusedMoEMethodBase:
return GptOssMxfp4MoEMethod(moe)
class GptOssMxfp4MoEMethod(FusedMoEMethodBase):
"""MXFP4 MoE quantization method."""
@@ -475,13 +483,45 @@ class GptOssMxfp4MoEMethod(FusedMoEMethodBase):
)
def _use_k3_situ_aiter(moe: FusedMoEConfig) -> bool:
"""Whether Kimi-K3's SiTU MXFP4 MoE should use the AITER A16W4 kernel.
K3 is weight-only MXFP4 (W4A16) with SiTU activation, which the generic
MXFP4 backend selector does not cover; route it to AITER on gfx950.
"""
from vllm.platforms import current_platform
if not current_platform.is_rocm():
return False
from vllm._aiter_ops import rocm_aiter_ops
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
from vllm.platforms.rocm import on_gfx950
return (
rocm_aiter_ops.is_fused_moe_enabled()
and on_gfx950()
and moe.activation == MoEActivation.SITU
and moe.activation_situ_linear_beta is not None
and rocm_aiter_ops.get_aiter_activation_type("situ") is not None
)
class Mxfp4MoEMethod(FusedMoEMethodBase):
"""MXFP4 MoE quantization method."""
def __init__(self, moe: FusedMoEConfig):
super().__init__(moe)
self.weight_dtype = "mxfp4"
self.mxfp4_backend, self.experts_cls = select_deepseek_v4_mxfp4_moe_backend(moe)
self.is_k3_situ_aiter = _use_k3_situ_aiter(moe)
self.experts_cls: type[mk.FusedMoEExperts] | None
if self.is_k3_situ_aiter:
self.mxfp4_backend = Mxfp4MoeBackend.AITER_MXFP4_BF16
self.experts_cls = backend_to_kernel_cls(self.mxfp4_backend)[0]
logger.info_once("Using AITER_MXFP4_BF16 for Kimi-K3 SiTU MXFP4 MoE.")
else:
self.mxfp4_backend, self.experts_cls = select_deepseek_v4_mxfp4_moe_backend(
moe
)
self.max_capture_size = moe.max_capture_size
@@ -523,6 +563,11 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
act_dtype=act_dtype,
moe_parallel_config=moe_parallel_config,
)
if self.is_k3_situ_aiter:
# K3's AITER A16W4 kernel handles K3's native intermediate size
# (moe_intermediate 3072; e.g. 384/partition at TP8); the generic
# 256 round-up would inflate weights and OOM.
return hidden_size, intermediate_size_per_partition
return mxfp4_round_up_hidden_size_and_intermediate_size(
self.mxfp4_backend, hidden_size, intermediate_size_per_partition
)
@@ -724,7 +769,58 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
layer=layer,
)
def _setup_kernel_k3_situ(self, layer: RoutedExperts) -> None:
# K3's AITER A16W4 kernel wants the separated ([gate_all, up_all])
# stage-1 layout, unlike the interleaved gpt-oss/DeepSeek path in
# convert_weight_to_mxfp4_moe_kernel_format. Preshuffle once here.
from aiter.utility.fp4_utils import e8m0_shuffle
from vllm._aiter_ops import rocm_aiter_ops
fp4_dtype = torch.float4_e2m1fn_x2
e8m0_dtype = torch.float8_e8m0fnu
num_experts = layer.w13_weight.shape[0]
# a8w4 (AITER_SITUV2_A8W4=1) uses the gate/up-interleaved (_gui_) fp8
# flydsl kernels, which need w13 weight+scale in interleave layout.
# Default a16w4 keeps the separated layout.
guinterleave = envs.AITER_SITUV2_A8W4
w13 = rocm_aiter_ops.shuffle_weight_a16w4(
layer.w13_weight.data.view(fp4_dtype), 16, guinterleave
)
w2 = rocm_aiter_ops.shuffle_weight_a16w4(
layer.w2_weight.data.view(fp4_dtype), 16, False
)
w13_scale_raw = layer.w13_weight_scale.data.view(e8m0_dtype)
w2_scale_raw = layer.w2_weight_scale.data.view(e8m0_dtype)
w13_scale = rocm_aiter_ops.shuffle_scale_a16w4(
w13_scale_raw.view(-1, w13_scale_raw.shape[-1]), num_experts, guinterleave
)
w2_scale = e8m0_shuffle(w2_scale_raw.view(-1, w2_scale_raw.shape[-1]))
replace_parameter(layer, "w13_weight", w13)
replace_parameter(layer, "w2_weight", w2)
replace_parameter(layer, "w13_weight_scale", w13_scale)
replace_parameter(layer, "w2_weight_scale", w2_scale)
layer.w13_weight.is_shuffled = True
layer.w2_weight.is_shuffled = True
self.moe_quant_config = self.get_fused_moe_quant_config(layer)
if self.moe_quant_config is not None and self.experts_cls is not None:
self.moe_kernel = make_mxfp4_moe_kernel(
moe_quant_config=self.moe_quant_config,
moe_config=self.moe,
mxfp4_backend=self.mxfp4_backend,
experts_cls=self.experts_cls,
routing_tables=layer._expert_routing_tables(),
layer=layer,
)
def process_weights_after_loading(self, layer):
if self.is_k3_situ_aiter:
self._setup_kernel_k3_situ(layer)
return
w13 = layer.w13_weight
w2 = layer.w2_weight
w13_scale = layer.w13_weight_scale
@@ -819,4 +915,8 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
global_num_experts=layer.global_num_experts,
expert_map=layer.expert_map,
apply_router_weight_on_input=layer.apply_router_weight_on_input,
num_expert_group=layer.num_expert_group,
topk_group=layer.topk_group,
e_score_correction_bias=layer.e_score_correction_bias,
routed_scaling_factor=layer.routed_scaling_factor,
)
@@ -18,20 +18,34 @@ def activation_to_flashinfer_int(activation: MoEActivation) -> int:
return activation_to_flashinfer_type(activation).value
def has_flashinfer_situ_activation() -> bool:
try:
from flashinfer.fused_moe.core import ActivationType
except ImportError:
return False
return hasattr(ActivationType, "Situ")
def activation_to_flashinfer_type(activation: MoEActivation) -> "ActivationType":
from flashinfer.fused_moe.core import ActivationType
if activation == MoEActivation.SITU:
situ = getattr(ActivationType, "Situ", None)
if situ is None:
raise ValueError("The installed FlashInfer does not support SITU")
return situ
# silu and gelu are mapped to their gated versions SwiGLU and GeGLU respectively
ACTIVATION_TO_FI_ACTIVATION = {
MoEActivation.SILU_NO_MUL: ActivationType.Silu,
MoEActivation.GELU_NO_MUL: ActivationType.Gelu,
MoEActivation.SILU: ActivationType.Swiglu,
# SwiGLU-OAI uses Swiglu; the OAI alpha/beta/clamp come from gemm1_* args.
MoEActivation.SWIGLUOAI: ActivationType.Swiglu,
MoEActivation.SWIGLUOAI_UNINTERLEAVE: ActivationType.Swiglu,
MoEActivation.GELU: ActivationType.Geglu,
MoEActivation.GELU_TANH: ActivationType.Geglu,
MoEActivation.RELU2_NO_MUL: ActivationType.Relu2,
MoEActivation.SWIGLUOAI_UNINTERLEAVE: ActivationType.Swiglu,
}
return ACTIVATION_TO_FI_ACTIVATION[activation]
@@ -474,7 +474,11 @@ def prepare_moe_fp4_layer_for_marlin(
group_size = 16 if is_nvfp4 else 32
e = layer.moe_config.num_experts
# Use the per-rank (local) expert count: under expert parallelism the
# w13_weight/w2_weight tensors only hold this rank's experts (created with
# local_num_experts), whereas moe_config.num_experts is the global count.
# With no EP the two are equal, so the non-EP path is unchanged.
e = layer.moe_config.num_local_experts
k = layer.moe_config.hidden_dim
n = layer.moe_config.intermediate_size_per_partition
@@ -41,8 +41,8 @@ def _mxfp8_e4m3_quantize_torch(
) -> tuple[torch.Tensor, torch.Tensor]:
"""Naive MXFP8 quantization.
For each block of 32 elements along the last dimension, compute a
shared e8m0 scale (the biased exponent of the block-wise amax)
and quantize each element to float8_e4m3fn.
shared e8m0 scale that fits the block-wise amax into the finite
float8_e4m3fn range, and quantize each element to float8_e4m3fn.
Returns (quantized_values [same shape, fp8], scales uint8).
Scale shape depends on is_sf_swizzled_layout:
@@ -58,7 +58,8 @@ def _mxfp8_e4m3_quantize_torch(
amax = x_blocked.abs().amax(dim=-1)
amax = amax.clamp(min=torch.finfo(torch.float32).tiny)
scale_biased = torch.floor(torch.log2(amax)) + 127.0
fp8_max = torch.finfo(MXFP8_VALUE_DTYPE).max
scale_biased = torch.ceil(torch.log2(amax / fp8_max)) + 127.0
scale_biased = scale_biased.clamp(0, 254)
scales_uint8 = scale_biased.to(torch.uint8)