[New model] Kimi K3 (#50000)
Signed-off-by: zjy0516 <riverclouds.zhu@qq.com> Signed-off-by: Isotr0py <Isotr0py@outlook.com> Signed-off-by: Bugen Zhao <i@bugenzhao.com> Signed-off-by: Jee Jee Li <jeejeelee@inferact.ai> Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai> Signed-off-by: Yongye Zhu <zyy1102000@gmail.com> Signed-off-by: wzhao18 <wzhao18.sz@gmail.com> Co-authored-by: Isotr0py <Isotr0py@outlook.com> Co-authored-by: Thien Tran <gau.nernst@yahoo.com.sg> Co-authored-by: Bugen Zhao <i@bugenzhao.com> Co-authored-by: Yongye Zhu <zyy1102000@gmail.com> Co-authored-by: Ziming Huang <zelda.huanghuang@gmail.com> Co-authored-by: Roger Wang <hey@rogerw.io> Co-authored-by: Isotr0py <mozf@inferact.ai> Co-authored-by: aoshen02 <aoshen02@users.noreply.github.com> Co-authored-by: Woosuk Kwon <woosuk@inferact.ai> Co-authored-by: Jee Jee Li <jeejeelee@inferact.ai> Co-authored-by: aoshen02 <aoshen@inferact.ai> Co-authored-by: Summer Yang <girasoleyang@gmail.com> Co-authored-by: Kevin H. Luu <khluu000@gmail.com> Co-authored-by: Bowen Wang <abmfy@icloud.com> Co-authored-by: gnovack <novackgm@gmail.com> Co-authored-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai> Co-authored-by: xiaozhoupy <peiyuanzhou1994@gmail.com> Co-authored-by: Roy Wang <yasong.wang@inferact.ai> Co-authored-by: Jeff (Junze) Ma <93145857+majunze2001@users.noreply.github.com> Co-authored-by: Codex <codex@openai.com> Co-authored-by: Wei Zhao <51183510+wzhao18@users.noreply.github.com>
This commit is contained in:
co-authored by
Isotr0py
Thien Tran
Bugen Zhao
Yongye Zhu
Ziming Huang
Roger Wang
Isotr0py
aoshen02
Woosuk Kwon
Jee Jee Li
aoshen02
Summer Yang
Kevin H. Luu
Bowen Wang
gnovack
Nick Hill
Yifan Qiao
xiaozhoupy
Roy Wang
Jeff Ma
Codex
Wei Zhao
parent
0c64be8873
commit
aeeb36b1f1
@@ -272,6 +272,7 @@ from vllm.v1.attention.backends.mla.prefill import (
|
||||
)
|
||||
from vllm.v1.attention.backends.utils import (
|
||||
get_dcp_local_seq_lens,
|
||||
get_num_attention_heads_from_layers,
|
||||
split_decodes_and_prefills,
|
||||
)
|
||||
from vllm.v1.attention.ops.common import cp_lse_ag_out_ar, cp_lse_ag_out_rs
|
||||
@@ -375,6 +376,7 @@ class MLAAttention(nn.Module, AttentionLayerBase):
|
||||
use_sparse: bool = False,
|
||||
indexer: object | None = None,
|
||||
topk_indices_buffer: torch.Tensor | None = None,
|
||||
non_causal_multi_token_decode: bool = False,
|
||||
**extra_impl_args,
|
||||
):
|
||||
super().__init__()
|
||||
@@ -391,6 +393,7 @@ class MLAAttention(nn.Module, AttentionLayerBase):
|
||||
self.head_size = kv_lora_rank + qk_rope_head_dim
|
||||
self.layer_name = prefix
|
||||
self.indexer = indexer
|
||||
self.non_causal_multi_token_decode = non_causal_multi_token_decode
|
||||
self.num_kv_heads = 1
|
||||
self.qk_head_dim = self.qk_nope_head_dim + self.qk_rope_head_dim
|
||||
|
||||
@@ -1123,6 +1126,7 @@ class MLAAttention(nn.Module, AttentionLayerBase):
|
||||
dtype=kv_cache_dtype,
|
||||
cache_dtype_str=self.kv_cache_dtype,
|
||||
kv_quant_mode=get_kv_quant_mode(self.kv_cache_dtype),
|
||||
non_causal_multi_token_decode=self.non_causal_multi_token_decode,
|
||||
)
|
||||
|
||||
def _v_up_proj(self, x: torch.Tensor, out: torch.Tensor):
|
||||
@@ -1450,6 +1454,8 @@ class MLACommonMetadata(AttentionMetadata, Generic[D]):
|
||||
num_decode_tokens: int
|
||||
num_prefills: int
|
||||
|
||||
causal: bool = True
|
||||
|
||||
# The dimension of the attention heads
|
||||
head_dim: int | None = None
|
||||
|
||||
@@ -1724,6 +1730,9 @@ class MLACommonMetadataBuilder(AttentionMetadataBuilder[M]):
|
||||
# speculative decoding is enabled.
|
||||
query_len_support: ClassVar[QueryLenSupport] = QueryLenSupport.SINGLE_ONLY
|
||||
|
||||
# Whether this builder can flatten a non-causal query block into decode rows.
|
||||
supports_non_causal_multi_token_decode: ClassVar[bool] = False
|
||||
|
||||
# The threshold for reordering the batch into decode and prefill requests.
|
||||
# If > 1, the batch will be reordered such that requests with
|
||||
# query length <= threshold are classified as decode requests.
|
||||
@@ -1824,8 +1833,14 @@ class MLACommonMetadataBuilder(AttentionMetadataBuilder[M]):
|
||||
self.vllm_config = vllm_config
|
||||
self.device = device
|
||||
self.use_pcp = parallel_config.prefill_context_parallel_size > 1
|
||||
self.non_causal_multi_token_decode = getattr(
|
||||
kv_cache_spec, "non_causal_multi_token_decode", False
|
||||
)
|
||||
|
||||
self.num_heads = self.model_config.get_num_attention_heads(parallel_config)
|
||||
# A draft cache group can have a different head count from the target.
|
||||
self.num_heads = get_num_attention_heads_from_layers(
|
||||
vllm_config, layer_names
|
||||
) or self.model_config.get_num_attention_heads(parallel_config)
|
||||
self.mla_dims = get_mla_dims(self.model_config)
|
||||
self.aot_schedule = current_platform.is_cuda()
|
||||
|
||||
@@ -1951,14 +1966,42 @@ class MLACommonMetadataBuilder(AttentionMetadataBuilder[M]):
|
||||
seq_lens = common_attn_metadata.seq_lens
|
||||
dcp_local_seq_lens = common_attn_metadata.dcp_local_seq_lens
|
||||
|
||||
num_decodes, num_prefills, num_decode_tokens, num_prefill_tokens = (
|
||||
split_decodes_and_prefills(
|
||||
common_attn_metadata,
|
||||
decode_threshold=self.reorder_batch_threshold,
|
||||
require_uniform=(self.query_len_support != QueryLenSupport.VARLEN),
|
||||
treat_short_extends_as_decodes=not self.use_pcp,
|
||||
non_causal_decode = common_attn_metadata.causal is False
|
||||
if non_causal_decode:
|
||||
if not (
|
||||
self.supports_non_causal_multi_token_decode
|
||||
and self.non_causal_multi_token_decode
|
||||
):
|
||||
raise ValueError(
|
||||
"Non-causal multi-token MLA requires an explicitly supported "
|
||||
"attention group."
|
||||
)
|
||||
query_lens = query_start_loc_cpu[1:] - query_start_loc_cpu[:-1]
|
||||
num_active_reqs = int(torch.count_nonzero(query_lens > 0))
|
||||
uniform_active_queries = num_active_reqs > 0 and bool(
|
||||
torch.all(query_lens[:num_active_reqs] == query_lens[0])
|
||||
)
|
||||
trailing_graph_padding = bool(torch.all(query_lens[num_active_reqs:] == 0))
|
||||
if not (uniform_active_queries and trailing_graph_padding):
|
||||
raise ValueError(
|
||||
"Non-causal MLA requires a uniform query block; got query "
|
||||
f"lengths {query_lens.tolist()}."
|
||||
)
|
||||
# Use exact GPU sequence lengths instead of the prefill path's CPU
|
||||
# context-length upper bounds.
|
||||
num_decodes = num_reqs
|
||||
num_prefills = 0
|
||||
num_decode_tokens = num_tokens
|
||||
num_prefill_tokens = 0
|
||||
else:
|
||||
num_decodes, num_prefills, num_decode_tokens, num_prefill_tokens = (
|
||||
split_decodes_and_prefills(
|
||||
common_attn_metadata,
|
||||
decode_threshold=self.reorder_batch_threshold,
|
||||
require_uniform=(self.query_len_support != QueryLenSupport.VARLEN),
|
||||
treat_short_extends_as_decodes=not self.use_pcp,
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
assert num_decodes + num_prefills == num_reqs
|
||||
assert num_decode_tokens + num_prefill_tokens == num_tokens
|
||||
@@ -2049,6 +2092,7 @@ class MLACommonMetadataBuilder(AttentionMetadataBuilder[M]):
|
||||
num_decodes=num_decodes,
|
||||
num_decode_tokens=num_decode_tokens,
|
||||
num_prefills=num_prefills,
|
||||
causal=not non_causal_decode,
|
||||
prefill=prefill_metadata,
|
||||
decode=decode_metadata,
|
||||
)
|
||||
|
||||
@@ -505,7 +505,7 @@ def deepgemm_moe_permute(
|
||||
dtype=torch.int32,
|
||||
)
|
||||
else:
|
||||
aq_scale_out = torch.empty((M_sum, sf_k), device=device, dtype=torch.float32)
|
||||
aq_scale_out = torch.zeros((M_sum, sf_k), device=device, dtype=torch.float32)
|
||||
|
||||
# DeepGEMM uses negative values in m_indices (here expert_ids) to mark
|
||||
# completely invalid / padded blocks that should be skipped. We always
|
||||
|
||||
@@ -204,6 +204,7 @@ def persistent_masked_m_silu_mul_quant(
|
||||
dtype=ys_dtype,
|
||||
device=y.device,
|
||||
)
|
||||
y_s.zero_()
|
||||
|
||||
ceil_ue8m0 = quant_scale_fmt in [
|
||||
DeepGemmQuantScaleFMT.FLOAT32_CEIL_UE8M0,
|
||||
|
||||
@@ -671,10 +671,10 @@ class MarlinExpertsBase(mk.FusedMoEExpertsModular):
|
||||
|
||||
@staticmethod
|
||||
def _supports_parallel_config(moe_parallel_config: FusedMoEParallelConfig) -> bool:
|
||||
return not (
|
||||
moe_parallel_config.use_fi_nvl_two_sided_kernels
|
||||
or moe_parallel_config.use_fi_nvl_one_sided_kernels
|
||||
)
|
||||
# One-sided FI-NVL all2all pairs with MarlinExperts fine (the
|
||||
# compressed-tensors MXFP4 path runs this exact combo); only the
|
||||
# two-sided kernels are unsupported here.
|
||||
return not moe_parallel_config.use_fi_nvl_two_sided_kernels
|
||||
|
||||
@property
|
||||
def quant_type_id(self) -> int:
|
||||
@@ -1068,11 +1068,8 @@ class BatchedMarlinExperts(MarlinExpertsBase):
|
||||
return
|
||||
|
||||
num_experts, max_num_tokens = hidden_states.shape[:2]
|
||||
beta = activation_situ_beta
|
||||
beta = 1.0 if activation_situ_beta is None else activation_situ_beta
|
||||
linear_beta = activation_situ_linear_beta
|
||||
assert beta is not None, (
|
||||
"SITU requires activation_situ_beta from FusedMoEConfig"
|
||||
)
|
||||
torch.ops._C.masked_situ_and_mul(
|
||||
act_output.view(num_experts, max_num_tokens, -1),
|
||||
act_input.view(num_experts, max_num_tokens, -1),
|
||||
|
||||
@@ -5,6 +5,7 @@ from functools import lru_cache
|
||||
|
||||
import torch
|
||||
|
||||
import vllm.envs as envs
|
||||
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
@@ -262,6 +263,8 @@ def rocm_aiter_fused_experts(
|
||||
elif activation == MoEActivation.SWIGLUOAI_UNINTERLEAVE:
|
||||
activation_method = rocm_aiter_ops.get_aiter_activation_type("swiglu")
|
||||
activation_interleave = False
|
||||
elif activation == MoEActivation.SITU:
|
||||
activation_method = rocm_aiter_ops.get_aiter_activation_type("situ")
|
||||
else:
|
||||
raise ValueError(f"Unsupported activation: {activation}")
|
||||
|
||||
@@ -356,10 +359,12 @@ def rocm_aiter_fused_experts(
|
||||
# https://github.com/ROCm/aiter/blob/v0.1.13.post1/aiter/fused_moe.py#L1099
|
||||
# TODO: Revisit this once we bump AITER to 0.1.15 with padding fixes
|
||||
# for CK/FlyDSL MoE GEMM e.g. https://github.com/ROCm/aiter/pull/3401
|
||||
hidden_pad = hidden_pad // 128 * 128
|
||||
intermediate_pad = (
|
||||
intermediate_pad // 64 * 64 * (2 if moe_config.tp_size == 1 else 1)
|
||||
)
|
||||
# SITU's A16W4 FlyDSL kernel pads per gate/up half; pass through unrounded.
|
||||
if activation != MoEActivation.SITU:
|
||||
hidden_pad = hidden_pad // 128 * 128
|
||||
intermediate_pad = (
|
||||
intermediate_pad // 64 * 64 * (2 if moe_config.tp_size == 1 else 1)
|
||||
)
|
||||
|
||||
# https://github.com/ROCm/aiter/pull/3123 specialized the AITER stage1 GEMMs
|
||||
# for interleaved vs separated gate and up weights.
|
||||
@@ -370,7 +375,15 @@ def rocm_aiter_fused_experts(
|
||||
from aiter.ops.flydsl.moe_common import GateMode
|
||||
|
||||
gate_mode = ""
|
||||
if quant_config.use_mxfp4_w4a16:
|
||||
if activation == MoEActivation.SITU:
|
||||
# a8w4 (AITER_SITUV2_A8W4=1) uses the gate/up-interleaved (_gui_)
|
||||
# fp8 flydsl kernels; default a16w4 SiTU stays separated.
|
||||
gate_mode = (
|
||||
GateMode.INTERLEAVE.value
|
||||
if envs.AITER_SITUV2_A8W4
|
||||
else GateMode.SEPARATED.value
|
||||
)
|
||||
elif quant_config.use_mxfp4_w4a16:
|
||||
gate_mode = GateMode.INTERLEAVE.value
|
||||
elif activation_interleave is not None:
|
||||
gate_mode = (
|
||||
@@ -401,6 +414,8 @@ def rocm_aiter_fused_experts(
|
||||
bias1=quant_config.w1_bias if quant_config.use_mxfp4_w4a16 else None,
|
||||
bias2=quant_config.w2_bias if quant_config.use_mxfp4_w4a16 else None,
|
||||
moe_sorting_dispatch_policy=moe_sorting_dispatch_policy,
|
||||
beta=moe_config.activation_situ_beta,
|
||||
linear_beta=moe_config.activation_situ_linear_beta,
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -15,6 +15,10 @@ from vllm.model_executor.layers.fused_moe.topk_weight_and_reduce import (
|
||||
TopKWeightAndReduceNoOP,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.utils import trtllm_moe_pack_topk_ids_weights
|
||||
from vllm.model_executor.layers.quantization.utils.flashinfer_utils import (
|
||||
activation_to_flashinfer_int,
|
||||
has_flashinfer_situ_activation,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
QuantKey,
|
||||
kMxfp4Static,
|
||||
@@ -133,19 +137,16 @@ class TrtLlmMxfp4ExpertsBase:
|
||||
|
||||
@staticmethod
|
||||
def _supports_activation(activation: MoEActivation) -> bool:
|
||||
if activation == MoEActivation.SITU:
|
||||
return has_flashinfer_situ_activation()
|
||||
return activation in (
|
||||
MoEActivation.SWIGLUOAI,
|
||||
MoEActivation.SILU,
|
||||
MoEActivation.SITU,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _flashinfer_activation_type(activation: MoEActivation) -> int:
|
||||
from flashinfer.fused_moe.core import ActivationType
|
||||
|
||||
if activation == MoEActivation.SITU:
|
||||
return ActivationType.Situ.value
|
||||
return ActivationType.Swiglu.value
|
||||
return activation_to_flashinfer_int(activation)
|
||||
|
||||
@staticmethod
|
||||
def activation_format() -> mk.FusedMoEActivationFormat:
|
||||
@@ -315,6 +316,75 @@ class TrtLlmMxfp4ExpertsModular(TrtLlmMxfp4ExpertsBase, mk.FusedMoEExpertsModula
|
||||
output = (M, self.hidden_dim_unpadded)
|
||||
return (workspace1, workspace2, output)
|
||||
|
||||
def _max_supported_tokens(self, top_k: int, global_num_experts: int) -> int:
|
||||
"""Max tokens per kernel call before the batched-GEMM grid overflows.
|
||||
|
||||
The TRTLLM-Gen batched GEMM launches a static grid whose batch (Y)
|
||||
dimension is ``getMaxNumCtasInBatchDim(num_tokens, top_k, num_experts,
|
||||
tileTokensDim)`` and must stay <= 65535. Solving that for num_tokens
|
||||
with the smallest tile the kernel may pick (tileTokensDim=8, the runner
|
||||
default) gives a bound that is safe regardless of the tactic selected.
|
||||
Without it, large batches (e.g. Kimi-K3 top_k=16, EP16 profiling with
|
||||
131072 gathered tokens) overflow the grid and the GEMM launch fails.
|
||||
"""
|
||||
MAX_GRID_Y = 65535
|
||||
MIN_TILE_TOKENS_DIM = 8
|
||||
max_tokens = (MAX_GRID_Y - global_num_experts) * MIN_TILE_TOKENS_DIM // top_k
|
||||
return max(1, min(300000, max_tokens))
|
||||
|
||||
def _invoke_kernel(
|
||||
self,
|
||||
output: torch.Tensor,
|
||||
x_quant: torch.Tensor,
|
||||
x_scale: torch.Tensor | None,
|
||||
topk_ids: torch.Tensor,
|
||||
topk_weights: torch.Tensor,
|
||||
w1: torch.Tensor,
|
||||
w2: torch.Tensor,
|
||||
activation: MoEActivation,
|
||||
global_num_experts: int,
|
||||
local_num_experts: int,
|
||||
local_expert_offset: int,
|
||||
topk: int,
|
||||
) -> None:
|
||||
from flashinfer import trtllm_fp4_block_scale_routed_moe
|
||||
|
||||
packed_tensor = trtllm_moe_pack_topk_ids_weights(topk_ids, topk_weights)
|
||||
trtllm_fp4_block_scale_routed_moe(
|
||||
topk_ids=packed_tensor,
|
||||
routing_bias=None,
|
||||
hidden_states=x_quant,
|
||||
hidden_states_scale=x_scale,
|
||||
gemm1_weights=w1,
|
||||
gemm1_weights_scale=self.w1_scale,
|
||||
gemm1_bias=self.w1_bias,
|
||||
gemm1_alpha=self.gemm1_alpha,
|
||||
gemm1_beta=self.gemm1_beta,
|
||||
gemm1_clamp_limit=self.gemm1_clamp_limit,
|
||||
gemm2_weights=w2,
|
||||
gemm2_weights_scale=self.w2_scale,
|
||||
gemm2_bias=self.w2_bias,
|
||||
output1_scale_scalar=None,
|
||||
output1_scale_gate_scalar=None,
|
||||
output2_scale_scalar=None,
|
||||
num_experts=global_num_experts,
|
||||
top_k=topk,
|
||||
n_group=None,
|
||||
topk_group=None,
|
||||
intermediate_size=self.intermediate_size_per_partition,
|
||||
local_expert_offset=local_expert_offset,
|
||||
local_num_experts=local_num_experts,
|
||||
routed_scaling_factor=None,
|
||||
# Modular kernel receives pre-routed tokens, so routing is already
|
||||
# done. Use Renormalize as a safe default the TRTLLM kernel supports.
|
||||
routing_method_type=RoutingMethodType.Renormalize,
|
||||
do_finalize=True,
|
||||
enable_pdl=True,
|
||||
activation_type=self._flashinfer_activation_type(activation),
|
||||
output=output,
|
||||
tune_max_num_tokens=max(self.max_capture_size, 1),
|
||||
)
|
||||
|
||||
def apply(
|
||||
self,
|
||||
output: torch.Tensor,
|
||||
@@ -335,7 +405,6 @@ class TrtLlmMxfp4ExpertsModular(TrtLlmMxfp4ExpertsBase, mk.FusedMoEExpertsModula
|
||||
):
|
||||
topk = topk_ids.size(-1)
|
||||
local_num_experts = w1.size(0)
|
||||
intermediate_size = self.intermediate_size_per_partition
|
||||
local_expert_offset = self.moe_config.ep_rank * local_num_experts
|
||||
|
||||
if a1q_scale is not None:
|
||||
@@ -346,49 +415,27 @@ class TrtLlmMxfp4ExpertsModular(TrtLlmMxfp4ExpertsBase, mk.FusedMoEExpertsModula
|
||||
x_quant = hidden_states
|
||||
x_scale = None
|
||||
|
||||
# Pack topk ids and weights into format expected by the kernel.
|
||||
packed_tensor = trtllm_moe_pack_topk_ids_weights(topk_ids, topk_weights)
|
||||
|
||||
assert self.w1_scale is not None
|
||||
assert self.w2_scale is not None
|
||||
kwargs = {
|
||||
"topk_ids": packed_tensor,
|
||||
"routing_bias": None,
|
||||
"hidden_states": x_quant,
|
||||
"hidden_states_scale": x_scale,
|
||||
"gemm1_weights": w1,
|
||||
"gemm1_weights_scale": self.w1_scale,
|
||||
"gemm1_bias": self.w1_bias,
|
||||
"gemm1_alpha": self.gemm1_alpha,
|
||||
"gemm1_beta": self.gemm1_beta,
|
||||
"gemm1_clamp_limit": self.gemm1_clamp_limit,
|
||||
"gemm2_weights": w2,
|
||||
"gemm2_weights_scale": self.w2_scale,
|
||||
"gemm2_bias": self.w2_bias,
|
||||
"output1_scale_scalar": None,
|
||||
"output1_scale_gate_scalar": None,
|
||||
"output2_scale_scalar": None,
|
||||
"num_experts": global_num_experts,
|
||||
"top_k": topk,
|
||||
"n_group": None,
|
||||
"topk_group": None,
|
||||
"intermediate_size": intermediate_size,
|
||||
"local_expert_offset": local_expert_offset,
|
||||
"local_num_experts": local_num_experts,
|
||||
"routed_scaling_factor": None,
|
||||
# Modular kernel receives pre-routed tokens, so routing
|
||||
# is already done. Use Renormalize as a safe default that
|
||||
# the TRTLLM C++ kernel supports.
|
||||
"routing_method_type": RoutingMethodType.Renormalize,
|
||||
"do_finalize": True,
|
||||
"enable_pdl": True,
|
||||
"activation_type": self._flashinfer_activation_type(activation),
|
||||
"output": output,
|
||||
"tune_max_num_tokens": max(self.max_capture_size, 1),
|
||||
}
|
||||
|
||||
from flashinfer import trtllm_fp4_block_scale_routed_moe
|
||||
|
||||
trtllm_fp4_block_scale_routed_moe(**kwargs)
|
||||
# Chunk tokens so the batched-GEMM grid stays within CUDA limits.
|
||||
M = x_quant.size(0)
|
||||
chunk_size = self._max_supported_tokens(topk, global_num_experts)
|
||||
for start in range(0, M, chunk_size):
|
||||
end = min(start + chunk_size, M)
|
||||
self._invoke_kernel(
|
||||
output[start:end],
|
||||
x_quant[start:end],
|
||||
None if x_scale is None else x_scale[start:end],
|
||||
topk_ids[start:end],
|
||||
topk_weights[start:end],
|
||||
w1,
|
||||
w2,
|
||||
activation,
|
||||
global_num_experts,
|
||||
local_num_experts,
|
||||
local_expert_offset,
|
||||
topk,
|
||||
)
|
||||
|
||||
return output
|
||||
|
||||
@@ -22,6 +22,7 @@ from vllm.model_executor.layers.fused_moe.utils import (
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.flashinfer_utils import (
|
||||
activation_to_flashinfer_int,
|
||||
has_flashinfer_situ_activation,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
QuantKey,
|
||||
@@ -225,15 +226,15 @@ class TrtLlmNvFp4ExpertsBase:
|
||||
|
||||
@staticmethod
|
||||
def _supports_activation(activation: MoEActivation) -> bool:
|
||||
"""Supports SiLU, RELU^2 non-gated, GELU, clamped SwiGLU-OAI, and the
|
||||
private SituGLU (SITU) cubin."""
|
||||
"""Supports SITU only when the installed FlashInfer exposes it."""
|
||||
if activation == MoEActivation.SITU:
|
||||
return has_flashinfer_situ_activation()
|
||||
return activation in [
|
||||
MoEActivation.SILU,
|
||||
MoEActivation.RELU2_NO_MUL,
|
||||
MoEActivation.GELU,
|
||||
MoEActivation.GELU_TANH,
|
||||
MoEActivation.SWIGLUOAI_UNINTERLEAVE,
|
||||
MoEActivation.SITU,
|
||||
]
|
||||
|
||||
@staticmethod
|
||||
|
||||
@@ -648,7 +648,7 @@ def mxfp4_round_up_hidden_size_and_intermediate_size(
|
||||
else:
|
||||
hidden_size = round_up(hidden_size, 256)
|
||||
elif backend in TRTLLM_BACKENDS:
|
||||
intermediate_size = round_up(intermediate_size, 256)
|
||||
intermediate_size = round_up(intermediate_size, 128)
|
||||
hidden_size = round_up(hidden_size, 256)
|
||||
elif backend in (
|
||||
Mxfp4MoeBackend.FLASHINFER_CUTLASS_MXFP4_BF16,
|
||||
|
||||
@@ -102,7 +102,7 @@ class BatchedPrepareAndFinalize(mk.FusedMoEPrepareAndFinalizeModular):
|
||||
num_local_experts, self.max_num_tokens, hidden_dim
|
||||
)
|
||||
|
||||
b_a1_scale = torch.empty(scale_shape, dtype=torch.float32, device=a1.device)
|
||||
b_a1_scale = torch.zeros(scale_shape, dtype=torch.float32, device=a1.device)
|
||||
else:
|
||||
assert quant_config.a1_scale is None
|
||||
b_a1_scale = None
|
||||
|
||||
@@ -6,11 +6,6 @@ import vllm.envs as envs
|
||||
from vllm.config import get_current_vllm_config
|
||||
from vllm.distributed import tensor_model_parallel_all_reduce
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.fused_allreduce_gemma_rms_norm import (
|
||||
_AR_RESIDUAL_RMS_NORM,
|
||||
_can_use_flashinfer,
|
||||
flashinfer_trtllm_fused_allreduce_norm,
|
||||
)
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.utils.torch_utils import aux_stream, current_stream
|
||||
|
||||
@@ -82,25 +77,30 @@ class LatentMoERunner(MoERunner):
|
||||
)
|
||||
self._k3_latent_moe_tail_op = op
|
||||
|
||||
def _get_zero_residual(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
||||
def _get_zero_residual(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
max_token_num: int,
|
||||
) -> torch.Tensor:
|
||||
"""Read-only zero ``residual_in`` for the fused AR+RMSNorm kernel.
|
||||
|
||||
flashinfer requires a residual buffer even when there is no residual to
|
||||
add.
|
||||
"""
|
||||
numel = hidden_states.numel()
|
||||
buf = getattr(self, "_zero_residual", None)
|
||||
if (
|
||||
buf is None
|
||||
or buf.dtype != hidden_states.dtype
|
||||
or buf.device != hidden_states.device
|
||||
or buf.numel() < numel
|
||||
):
|
||||
if buf is None:
|
||||
buf = torch.zeros(
|
||||
numel, dtype=hidden_states.dtype, device=hidden_states.device
|
||||
max_token_num * hidden_states.shape[-1],
|
||||
dtype=hidden_states.dtype,
|
||||
device=hidden_states.device,
|
||||
)
|
||||
self._zero_residual = buf
|
||||
return buf[:numel].view_as(hidden_states)
|
||||
|
||||
assert buf.dtype == hidden_states.dtype
|
||||
assert buf.device == hidden_states.device
|
||||
assert hidden_states.numel() <= buf.numel()
|
||||
|
||||
return buf[: hidden_states.numel()].view_as(hidden_states)
|
||||
|
||||
def _use_fused_path(self) -> bool:
|
||||
# The fused path merges the latent and shared reductions into one
|
||||
@@ -224,6 +224,12 @@ class LatentMoERunner(MoERunner):
|
||||
norm: RMSNorm,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""All-reduce + add residual + (standard) RMSNorm, fused via flashinfer."""
|
||||
from vllm.model_executor.layers.fused_allreduce_gemma_rms_norm import (
|
||||
_AR_RESIDUAL_RMS_NORM,
|
||||
_can_use_flashinfer,
|
||||
flashinfer_trtllm_fused_allreduce_norm,
|
||||
)
|
||||
|
||||
if self.moe_config.tp_size == 1:
|
||||
return norm(hidden_states)
|
||||
|
||||
@@ -238,7 +244,7 @@ class LatentMoERunner(MoERunner):
|
||||
# buffer and the normalized result into norm_out.
|
||||
flashinfer_trtllm_fused_allreduce_norm(
|
||||
allreduce_in=hidden_states,
|
||||
residual=self._get_zero_residual(hidden_states),
|
||||
residual=self._get_zero_residual(hidden_states, max_token_num),
|
||||
rms_gamma=norm.weight,
|
||||
rms_eps=norm.variance_epsilon,
|
||||
world_size=self.moe_config.tp_size,
|
||||
|
||||
@@ -1,32 +1,33 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from collections.abc import Callable
|
||||
|
||||
import torch
|
||||
from einops import rearrange
|
||||
from torch import nn
|
||||
from torch.nn.parameter import Parameter
|
||||
|
||||
from vllm.config import VllmConfig, get_current_vllm_config
|
||||
from vllm.distributed import (
|
||||
divide,
|
||||
)
|
||||
from vllm.forward_context import ForwardContext, get_forward_context
|
||||
from vllm.logger import init_logger
|
||||
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed import divide, get_tensor_model_parallel_rank
|
||||
from vllm.forward_context import get_forward_context
|
||||
from vllm.model_executor.custom_op import PluggableLayer
|
||||
from vllm.model_executor.layers.mamba.gdn.base import GatedDeltaNetAttention
|
||||
from vllm.model_executor.model_loader.weight_utils import sharded_weight_loader
|
||||
from vllm.model_executor.utils import set_weight_attrs
|
||||
from vllm.third_party.flash_linear_attention.ops.kda import (
|
||||
FusedRMSNormGated,
|
||||
chunk_kda_with_fused_gate,
|
||||
fused_kda_gate,
|
||||
fused_recurrent_kda,
|
||||
from vllm.model_executor.model_loader.weight_utils import (
|
||||
default_weight_loader,
|
||||
sharded_weight_loader,
|
||||
)
|
||||
from vllm.model_executor.parameter import BasevLLMParameter
|
||||
from vllm.model_executor.utils import set_weight_attrs
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.third_party.flash_linear_attention.ops.kda import FusedRMSNormGated
|
||||
from vllm.transformers_utils.configs.kimi_linear import KimiLinearConfig
|
||||
from vllm.utils.torch_utils import direct_register_custom_op
|
||||
from vllm.v1.attention.backends.gdn_attn import GDNAttentionMetadata
|
||||
|
||||
from ...linear import (
|
||||
ColumnParallelLinear,
|
||||
MergedColumnParallelLinear,
|
||||
ReplicatedLinear,
|
||||
RowParallelLinear,
|
||||
)
|
||||
@@ -36,49 +37,118 @@ from ..mamba_utils import (
|
||||
is_conv_state_dim_first,
|
||||
)
|
||||
from ..ops.causal_conv1d import causal_conv1d_fn, causal_conv1d_update
|
||||
from ..ops.gather_initial_states import gather_initial_states
|
||||
|
||||
logger = init_logger(__name__)
|
||||
# Empirical lower bound for the KDA gate to avoid numerical underflow.
|
||||
_KDA_GATE_LOGBOUND_MIN = -5.0
|
||||
|
||||
|
||||
def kda_attention(
|
||||
q_proj_states: torch.Tensor,
|
||||
k_proj_states: torch.Tensor,
|
||||
v_proj_states: torch.Tensor,
|
||||
g1: torch.Tensor,
|
||||
beta: torch.Tensor,
|
||||
core_attn_out: torch.Tensor,
|
||||
layer_name: str,
|
||||
) -> None:
|
||||
forward_context: ForwardContext = get_forward_context()
|
||||
self = forward_context.no_compile_layers[layer_name]
|
||||
self._forward(
|
||||
q_proj_states=q_proj_states,
|
||||
k_proj_states=k_proj_states,
|
||||
v_proj_states=v_proj_states,
|
||||
g1=g1,
|
||||
beta=beta,
|
||||
core_attn_out=core_attn_out,
|
||||
)
|
||||
def a_log_weight_loader(
|
||||
shard_axis: int,
|
||||
) -> Callable[[torch.Tensor, torch.Tensor], None]:
|
||||
"""Load KDA A_log stored as either old 4D or current 1D weights."""
|
||||
|
||||
def loader(param: torch.Tensor, loaded_weight: torch.Tensor) -> None:
|
||||
tp_rank = get_tensor_model_parallel_rank()
|
||||
shard_size = param.data.shape[shard_axis]
|
||||
start_idx = tp_rank * shard_size
|
||||
|
||||
if loaded_weight.dim() == 4:
|
||||
assert loaded_weight.shape[:2] == (1, 1), (
|
||||
f"Expected old A_log shape (1, 1, H, 1), got {loaded_weight.shape}"
|
||||
)
|
||||
assert loaded_weight.shape[-1] == 1, (
|
||||
f"Expected old A_log last dim to be 1, got {loaded_weight.shape}"
|
||||
)
|
||||
loaded_weight = loaded_weight.view(loaded_weight.shape[2])
|
||||
|
||||
loaded_weight = loaded_weight.narrow(shard_axis, start_idx, shard_size)
|
||||
return default_weight_loader(param, loaded_weight)
|
||||
|
||||
return loader
|
||||
|
||||
|
||||
def kda_attention_fake(
|
||||
q_proj_states: torch.Tensor,
|
||||
k_proj_states: torch.Tensor,
|
||||
v_proj_states: torch.Tensor,
|
||||
g1: torch.Tensor,
|
||||
beta: torch.Tensor,
|
||||
core_attn_out: torch.Tensor,
|
||||
layer_name: str,
|
||||
) -> None:
|
||||
return
|
||||
def _make_fused_conv1d_weight_loader(
|
||||
dims: list[int],
|
||||
tp_size: int,
|
||||
tp_rank: int,
|
||||
) -> Callable[..., None]:
|
||||
sharded_dims = [dim // tp_size for dim in dims]
|
||||
|
||||
def weight_loader(
|
||||
param: torch.Tensor,
|
||||
loaded_weight: torch.Tensor,
|
||||
loaded_shard_id: int,
|
||||
) -> None:
|
||||
if loaded_weight.dim() == 2:
|
||||
loaded_weight = loaded_weight.unsqueeze(1)
|
||||
shard_size = sharded_dims[loaded_shard_id]
|
||||
source_start = tp_rank * shard_size
|
||||
target_start = sum(sharded_dims[:loaded_shard_id])
|
||||
loaded_shard = loaded_weight[source_start : source_start + shard_size]
|
||||
param.data[target_start : target_start + shard_size].copy_(loaded_shard)
|
||||
|
||||
return weight_loader
|
||||
|
||||
|
||||
direct_register_custom_op(
|
||||
op_name="kda_attention",
|
||||
op_func=kda_attention,
|
||||
mutates_args=["core_attn_out"],
|
||||
fake_impl=kda_attention_fake,
|
||||
)
|
||||
class _KimiGDNMergedColumnParallelLinear(MergedColumnParallelLinear):
|
||||
"""Merged projection with one output replicated across TP ranks.
|
||||
|
||||
The replicated shard is represented as ``size * tp_size`` so the merged
|
||||
parameter reserves ``size`` local rows on every rank. Loading that shard
|
||||
from rank zero then gives every rank the complete checkpoint weight.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
input_size: int,
|
||||
output_sizes: list[int],
|
||||
replicated_shard_id: int,
|
||||
tp_size: int,
|
||||
**kwargs,
|
||||
) -> None:
|
||||
self.replicated_shard_id = replicated_shard_id
|
||||
output_sizes = output_sizes.copy()
|
||||
output_sizes[replicated_shard_id] *= tp_size
|
||||
super().__init__(input_size, output_sizes, **kwargs)
|
||||
|
||||
def weight_loader(
|
||||
self,
|
||||
param: Parameter,
|
||||
loaded_weight: torch.Tensor,
|
||||
loaded_shard_id: tuple[int, ...] | int | None = None,
|
||||
) -> None:
|
||||
tp_rank = self.tp_rank
|
||||
param_tp_rank = getattr(param, "tp_rank", None)
|
||||
if loaded_shard_id == self.replicated_shard_id:
|
||||
self.tp_rank = 0
|
||||
if param_tp_rank is not None:
|
||||
param.tp_rank = 0
|
||||
try:
|
||||
super().weight_loader(param, loaded_weight, loaded_shard_id)
|
||||
finally:
|
||||
self.tp_rank = tp_rank
|
||||
if param_tp_rank is not None:
|
||||
param.tp_rank = param_tp_rank
|
||||
|
||||
def weight_loader_v2(
|
||||
self,
|
||||
param: BasevLLMParameter,
|
||||
loaded_weight: torch.Tensor,
|
||||
loaded_shard_id: tuple[int, ...] | int | None = None,
|
||||
) -> None:
|
||||
tp_rank = self.tp_rank
|
||||
param_tp_rank = getattr(param, "tp_rank", None)
|
||||
if loaded_shard_id == self.replicated_shard_id:
|
||||
self.tp_rank = 0
|
||||
if param_tp_rank is not None:
|
||||
param.tp_rank = 0
|
||||
try:
|
||||
super().weight_loader_v2(param, loaded_weight, loaded_shard_id)
|
||||
finally:
|
||||
self.tp_rank = tp_rank
|
||||
if param_tp_rank is not None:
|
||||
param.tp_rank = param_tp_rank
|
||||
|
||||
|
||||
@PluggableLayer.register("kimi_gated_delta_net_attention")
|
||||
@@ -96,7 +166,11 @@ class KimiGatedDeltaNetAttention(GatedDeltaNetAttention):
|
||||
self,
|
||||
) -> tuple[tuple[int, ...], tuple[int, ...]]:
|
||||
return MambaStateShapeCalculator.kda_state_shape(
|
||||
self.tp_size, self.num_heads, self.head_dim, conv_kernel_size=self.conv_size
|
||||
self.tp_size,
|
||||
self.num_heads,
|
||||
self.head_dim,
|
||||
conv_kernel_size=self.conv_size,
|
||||
num_spec=self.num_spec,
|
||||
)
|
||||
|
||||
def __init__(
|
||||
@@ -114,122 +188,142 @@ class KimiGatedDeltaNetAttention(GatedDeltaNetAttention):
|
||||
assert self.num_heads % self.tp_size == 0
|
||||
self.local_num_heads = divide(self.num_heads, self.tp_size)
|
||||
|
||||
projection_size = self.head_dim * self.num_heads
|
||||
self.projection_size = self.head_dim * self.num_heads
|
||||
self.local_projection_size = divide(self.projection_size, self.tp_size)
|
||||
self.conv_size = kda_config["short_conv_kernel_size"]
|
||||
self.use_full_rank_gate = kda_config.get("use_full_rank_gate", False)
|
||||
|
||||
self.q_proj = ColumnParallelLinear(
|
||||
if self.use_full_rank_gate:
|
||||
# Keep f_a before the narrow beta shard, then pad each TP-local row
|
||||
# to select the aligned BF16 GEMM path. The padding also avoids an
|
||||
# Inductor correctness issue seen with the row-strided G view.
|
||||
qkvg_output_sizes = [self.projection_size] * 4
|
||||
in_proj_output_sizes = qkvg_output_sizes + [
|
||||
self.head_dim,
|
||||
self.num_heads,
|
||||
]
|
||||
local_output_size = (
|
||||
4 * self.local_projection_size + self.head_dim + self.local_num_heads
|
||||
)
|
||||
self.in_proj_padding = -local_output_size % 16
|
||||
if self.in_proj_padding:
|
||||
in_proj_output_sizes.append(self.in_proj_padding * self.tp_size)
|
||||
else:
|
||||
in_proj_output_sizes = [self.projection_size] * 3 + [
|
||||
self.num_heads,
|
||||
self.head_dim,
|
||||
]
|
||||
self.in_proj_padding = 0
|
||||
self.in_proj_qkvgfab = _KimiGDNMergedColumnParallelLinear(
|
||||
self.hidden_size,
|
||||
projection_size,
|
||||
in_proj_output_sizes,
|
||||
replicated_shard_id=4,
|
||||
tp_size=self.tp_size,
|
||||
bias=False,
|
||||
quant_config=self.quant_config,
|
||||
prefix=f"{prefix}.q_proj",
|
||||
)
|
||||
self.k_proj = ColumnParallelLinear(
|
||||
self.hidden_size,
|
||||
projection_size,
|
||||
bias=False,
|
||||
quant_config=self.quant_config,
|
||||
prefix=f"{prefix}.k_proj",
|
||||
)
|
||||
self.v_proj = ColumnParallelLinear(
|
||||
self.hidden_size,
|
||||
projection_size,
|
||||
bias=False,
|
||||
quant_config=self.quant_config,
|
||||
prefix=f"{prefix}.v_proj",
|
||||
)
|
||||
|
||||
self.f_a_proj = ReplicatedLinear(
|
||||
self.hidden_size,
|
||||
self.head_dim,
|
||||
bias=False,
|
||||
quant_config=self.quant_config,
|
||||
prefix=f"{prefix}.f_a_proj",
|
||||
prefix=f"{prefix}.in_proj_qkvgfab",
|
||||
)
|
||||
if self.in_proj_padding:
|
||||
self.in_proj_qkvgfab.weight.data[-self.in_proj_padding :].zero_()
|
||||
|
||||
self.f_b_proj = ColumnParallelLinear(
|
||||
self.head_dim,
|
||||
projection_size,
|
||||
self.projection_size,
|
||||
bias=False,
|
||||
quant_config=self.quant_config,
|
||||
prefix=f"{prefix}.f_b_proj",
|
||||
)
|
||||
self.dt_bias = nn.Parameter(
|
||||
torch.empty(divide(projection_size, self.tp_size), dtype=torch.float32)
|
||||
torch.empty(self.local_projection_size, dtype=torch.float32)
|
||||
)
|
||||
|
||||
set_weight_attrs(self.dt_bias, {"weight_loader": sharded_weight_loader(0)})
|
||||
|
||||
self.b_proj = ColumnParallelLinear(
|
||||
self.hidden_size,
|
||||
self.num_heads,
|
||||
bias=False,
|
||||
quant_config=self.quant_config,
|
||||
prefix=f"{prefix}.b_proj",
|
||||
)
|
||||
|
||||
self.q_conv1d = ColumnParallelLinear(
|
||||
# One packed parameter and cache let decode run a single conv update.
|
||||
# Prefill slices them back into Q/K/V to obtain dense outputs cheaply.
|
||||
self.conv1d = ColumnParallelLinear(
|
||||
input_size=self.conv_size,
|
||||
output_size=projection_size,
|
||||
output_size=3 * self.projection_size,
|
||||
bias=False,
|
||||
params_dtype=torch.float32,
|
||||
prefix=f"{prefix}.q_conv1d",
|
||||
prefix=f"{prefix}.conv1d",
|
||||
)
|
||||
self.k_conv1d = ColumnParallelLinear(
|
||||
input_size=self.conv_size,
|
||||
output_size=projection_size,
|
||||
bias=False,
|
||||
params_dtype=torch.float32,
|
||||
prefix=f"{prefix}.k_conv1d",
|
||||
self.conv1d.weight.data = self.conv1d.weight.data.unsqueeze(1)
|
||||
delattr(self.conv1d.weight, "weight_loader")
|
||||
set_weight_attrs(
|
||||
self.conv1d.weight,
|
||||
{
|
||||
"weight_loader": _make_fused_conv1d_weight_loader(
|
||||
[self.projection_size] * 3,
|
||||
self.tp_size,
|
||||
self.tp_rank,
|
||||
)
|
||||
},
|
||||
)
|
||||
self.v_conv1d = ColumnParallelLinear(
|
||||
input_size=self.conv_size,
|
||||
output_size=projection_size,
|
||||
bias=False,
|
||||
params_dtype=torch.float32,
|
||||
prefix=f"{prefix}.v_conv1d",
|
||||
)
|
||||
# unsqueeze to fit conv1d weights shape into the linear weights shape.
|
||||
# Can't do this in `weight_loader` since it already exists in
|
||||
# `ColumnParallelLinear` and `set_weight_attrs`
|
||||
# doesn't allow to override it
|
||||
self.q_conv1d.weight.data = self.q_conv1d.weight.data.unsqueeze(1)
|
||||
self.k_conv1d.weight.data = self.k_conv1d.weight.data.unsqueeze(1)
|
||||
self.v_conv1d.weight.data = self.v_conv1d.weight.data.unsqueeze(1)
|
||||
|
||||
self.A_log = nn.Parameter(
|
||||
torch.empty(1, 1, self.local_num_heads, 1, dtype=torch.float32)
|
||||
torch.empty(self.local_num_heads, dtype=torch.float32)
|
||||
)
|
||||
set_weight_attrs(self.A_log, {"weight_loader": sharded_weight_loader(2)})
|
||||
set_weight_attrs(self.A_log, {"weight_loader": a_log_weight_loader(0)})
|
||||
|
||||
self.g_a_proj = ReplicatedLinear(
|
||||
self.hidden_size,
|
||||
self.head_dim,
|
||||
bias=False,
|
||||
quant_config=self.quant_config,
|
||||
prefix=f"{prefix}.g_a_proj",
|
||||
self.gate_lower_bound: float | None = kda_config.get("gate_lower_bound", None)
|
||||
if self.gate_lower_bound is not None:
|
||||
assert _KDA_GATE_LOGBOUND_MIN <= self.gate_lower_bound < 0, (
|
||||
"KDA gate lower bound must be in "
|
||||
f"[{_KDA_GATE_LOGBOUND_MIN}, 0). "
|
||||
f"Got {self.gate_lower_bound}."
|
||||
)
|
||||
self.use_safe_gate = self.gate_lower_bound is not None
|
||||
additional_config = vllm_config.additional_config
|
||||
backend = (
|
||||
additional_config.get("kda_prefill_backend", "auto")
|
||||
if isinstance(additional_config, dict)
|
||||
else "auto"
|
||||
)
|
||||
self.g_b_proj = ColumnParallelLinear(
|
||||
self.head_dim,
|
||||
projection_size,
|
||||
bias=False,
|
||||
quant_config=self.quant_config,
|
||||
prefix=f"{prefix}.g_b_proj",
|
||||
backend = "triton" if backend == "auto" else backend
|
||||
assert backend == "triton", (
|
||||
"The shared Kimi GDN layer only supports the Triton KDA "
|
||||
f"prefill backend, got {backend!r}."
|
||||
)
|
||||
if not self.use_full_rank_gate:
|
||||
self.g_a_proj = ReplicatedLinear(
|
||||
self.hidden_size,
|
||||
self.head_dim,
|
||||
bias=False,
|
||||
quant_config=self.quant_config,
|
||||
prefix=f"{prefix}.g_a_proj",
|
||||
)
|
||||
self.g_b_proj = ColumnParallelLinear(
|
||||
self.head_dim,
|
||||
self.projection_size,
|
||||
bias=False,
|
||||
quant_config=self.quant_config,
|
||||
prefix=f"{prefix}.g_b_proj",
|
||||
)
|
||||
self.o_norm = FusedRMSNormGated(self.head_dim, activation="sigmoid")
|
||||
self.o_proj = RowParallelLinear(
|
||||
projection_size,
|
||||
self.projection_size,
|
||||
self.hidden_size,
|
||||
bias=False,
|
||||
quant_config=self.quant_config,
|
||||
prefix=f"{prefix}.o_proj",
|
||||
)
|
||||
|
||||
compilation_config = get_current_vllm_config().compilation_config
|
||||
compilation_config = vllm_config.compilation_config
|
||||
if prefix in compilation_config.static_forward_context:
|
||||
raise ValueError(f"Duplicate layer name: {prefix}")
|
||||
compilation_config.static_forward_context[prefix] = self
|
||||
|
||||
def rearrange_mixed_qkv(
|
||||
self, mixed_qkv: torch.Tensor
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
seq_len = mixed_qkv.shape[0]
|
||||
qkv = mixed_qkv.view(seq_len, 3, self.local_num_heads, self.head_dim)
|
||||
# Materialize all three row-strided inputs with one token-major to
|
||||
# QKV-major permutation. Each unbound tensor is then contiguous.
|
||||
qkv = qkv.permute(1, 0, 2, 3).contiguous().unsqueeze(1)
|
||||
return qkv.unbind(0)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
@@ -237,42 +331,57 @@ class KimiGatedDeltaNetAttention(GatedDeltaNetAttention):
|
||||
output: torch.Tensor,
|
||||
) -> None:
|
||||
num_tokens = hidden_states.size(0)
|
||||
q = self.q_proj(hidden_states)[0]
|
||||
k = self.k_proj(hidden_states)[0]
|
||||
v = self.v_proj(hidden_states)[0]
|
||||
projected_qkvgfab = self.in_proj_qkvgfab(hidden_states)[0]
|
||||
if self.use_full_rank_gate:
|
||||
split_sizes = [
|
||||
3 * self.local_projection_size,
|
||||
self.local_projection_size,
|
||||
self.head_dim,
|
||||
self.local_num_heads,
|
||||
]
|
||||
if self.in_proj_padding:
|
||||
split_sizes.append(self.in_proj_padding)
|
||||
projected = projected_qkvgfab.split(split_sizes, dim=-1)
|
||||
mixed_qkv, g_proj_states, f_a, beta = projected[:4]
|
||||
else:
|
||||
mixed_qkv, beta, f_a = projected_qkvgfab.split(
|
||||
[
|
||||
3 * self.local_projection_size,
|
||||
self.local_num_heads,
|
||||
self.head_dim,
|
||||
],
|
||||
dim=-1,
|
||||
)
|
||||
g_proj_states = self.g_b_proj(self.g_a_proj(hidden_states)[0])[0]
|
||||
|
||||
beta = self.b_proj(hidden_states)[0].float().sigmoid()
|
||||
g1 = self.f_b_proj(self.f_a_proj(hidden_states)[0])[0]
|
||||
g1 = self.f_b_proj(f_a)[0]
|
||||
beta = beta.unsqueeze(0)
|
||||
g1 = rearrange(g1, "n (h d) -> 1 n h d", d=self.head_dim)
|
||||
|
||||
g_proj_states = self.g_b_proj(self.g_a_proj(hidden_states)[0])[0]
|
||||
g2 = rearrange(g_proj_states, "... (h d) -> ... h d", d=self.head_dim)
|
||||
|
||||
core_attn_out = torch.zeros(
|
||||
core_attn_out = torch.empty(
|
||||
(1, num_tokens, self.local_num_heads, self.head_dim),
|
||||
dtype=hidden_states.dtype,
|
||||
device=hidden_states.device,
|
||||
)
|
||||
torch.ops.vllm.kda_attention(
|
||||
q,
|
||||
k,
|
||||
v,
|
||||
g1,
|
||||
beta,
|
||||
core_attn_out,
|
||||
self.prefix,
|
||||
|
||||
self._forward(
|
||||
mixed_qkv=mixed_qkv,
|
||||
g1=g1,
|
||||
g2=g2,
|
||||
beta=beta,
|
||||
core_attn_out=core_attn_out,
|
||||
)
|
||||
core_attn_out = self.o_norm(core_attn_out, g2)
|
||||
core_attn_out = rearrange(core_attn_out, "1 n h d -> n (h d)")
|
||||
output[:] = self.o_proj(core_attn_out)[0]
|
||||
|
||||
@eager_break_during_capture
|
||||
def _forward(
|
||||
self,
|
||||
q_proj_states: torch.Tensor,
|
||||
k_proj_states: torch.Tensor,
|
||||
v_proj_states: torch.Tensor,
|
||||
mixed_qkv: torch.Tensor,
|
||||
g1: torch.Tensor,
|
||||
g2: torch.Tensor,
|
||||
beta: torch.Tensor,
|
||||
core_attn_out: torch.Tensor,
|
||||
) -> None:
|
||||
@@ -280,165 +389,247 @@ class KimiGatedDeltaNetAttention(GatedDeltaNetAttention):
|
||||
attn_metadata_raw = forward_context.attn_metadata
|
||||
|
||||
if attn_metadata_raw is None:
|
||||
# # V1 profile run
|
||||
return
|
||||
|
||||
# Vendor-specific KDA kernels: AMD/ROCm and NVIDIA keep their own copies
|
||||
# under kimi_k3/{amd,nvidia}/ops so each can diverge independently.
|
||||
if current_platform.is_rocm():
|
||||
from vllm.models.kimi_k3.amd.ops.third_party.kda import (
|
||||
chunk_kda_with_fused_gate,
|
||||
fused_recurrent_kda,
|
||||
fused_recurrent_kda_packed_decode,
|
||||
)
|
||||
else:
|
||||
from vllm.models.kimi_k3.nvidia.ops.third_party.kda import (
|
||||
chunk_kda_with_fused_gate,
|
||||
fused_recurrent_kda,
|
||||
fused_recurrent_kda_packed_decode,
|
||||
)
|
||||
|
||||
assert isinstance(attn_metadata_raw, dict)
|
||||
attn_metadata_narrowed = attn_metadata_raw[self.prefix]
|
||||
assert isinstance(attn_metadata_narrowed, GDNAttentionMetadata)
|
||||
has_initial_state = attn_metadata_narrowed.has_initial_state
|
||||
non_spec_query_start_loc = attn_metadata_narrowed.non_spec_query_start_loc
|
||||
non_spec_state_indices_tensor = (
|
||||
attn_metadata_narrowed.non_spec_state_indices_tensor
|
||||
) # noqa: E501
|
||||
num_actual_tokens = attn_metadata_narrowed.num_actual_tokens
|
||||
constant_caches = self.kv_cache
|
||||
|
||||
q_proj_states = q_proj_states[:num_actual_tokens]
|
||||
k_proj_states = k_proj_states[:num_actual_tokens]
|
||||
v_proj_states = v_proj_states[:num_actual_tokens]
|
||||
m = attn_metadata_narrowed
|
||||
has_initial_state = m.has_initial_state
|
||||
non_spec_query_start_loc = m.non_spec_query_start_loc
|
||||
non_spec_state_indices_tensor = m.non_spec_state_indices_tensor
|
||||
spec_sequence_masks = m.spec_sequence_masks
|
||||
spec_token_indx = m.spec_token_indx
|
||||
non_spec_token_indx = m.non_spec_token_indx
|
||||
spec_state_indices_tensor = m.spec_state_indices_tensor
|
||||
spec_query_start_loc = m.spec_query_start_loc
|
||||
num_accepted_tokens = m.num_accepted_tokens
|
||||
num_actual_tokens = m.num_actual_tokens
|
||||
mixed_qkv = mixed_qkv[:num_actual_tokens]
|
||||
g1 = g1[:, :num_actual_tokens]
|
||||
beta = beta[:, :num_actual_tokens]
|
||||
|
||||
(conv_state, recurrent_state) = constant_caches
|
||||
constant_caches = self.kv_cache
|
||||
|
||||
conv_state, recurrent_state = constant_caches
|
||||
# conv_state must be (..., dim, width-1) for the conv kernels.
|
||||
# DS layout stores it that way directly; SD layout needs a transpose.
|
||||
if not is_conv_state_dim_first():
|
||||
conv_state = conv_state.transpose(-1, -2)
|
||||
|
||||
conv_state_q, conv_state_k, conv_state_v = conv_state.chunk(3, dim=-2)
|
||||
conv_weights = self.conv1d.weight.view(
|
||||
self.conv1d.weight.size(0), self.conv1d.weight.size(2)
|
||||
)
|
||||
q_conv_weight, k_conv_weight, v_conv_weight = conv_weights.split(
|
||||
self.local_projection_size, dim=0
|
||||
)
|
||||
q_conv_state, k_conv_state, v_conv_state = conv_state.split(
|
||||
self.local_projection_size, dim=-2
|
||||
)
|
||||
|
||||
q_conv_weights = self.q_conv1d.weight.view(
|
||||
self.q_conv1d.weight.size(0), self.q_conv1d.weight.size(2)
|
||||
)
|
||||
k_conv_weights = self.k_conv1d.weight.view(
|
||||
self.k_conv1d.weight.size(0), self.k_conv1d.weight.size(2)
|
||||
)
|
||||
v_conv_weights = self.v_conv1d.weight.view(
|
||||
self.v_conv1d.weight.size(0), self.v_conv1d.weight.size(2)
|
||||
)
|
||||
if attn_metadata_narrowed.num_prefills > 0:
|
||||
q_proj_states = q_proj_states.transpose(0, 1)
|
||||
k_proj_states = k_proj_states.transpose(0, 1)
|
||||
v_proj_states = v_proj_states.transpose(0, 1)
|
||||
q = causal_conv1d_fn(
|
||||
q_proj_states,
|
||||
q_conv_weights,
|
||||
self.q_conv1d.bias,
|
||||
activation="silu",
|
||||
conv_states=conv_state_q,
|
||||
has_initial_state=has_initial_state,
|
||||
cache_indices=non_spec_state_indices_tensor,
|
||||
query_start_loc=non_spec_query_start_loc,
|
||||
metadata=attn_metadata_narrowed,
|
||||
).transpose(0, 1)
|
||||
k = causal_conv1d_fn(
|
||||
k_proj_states,
|
||||
k_conv_weights,
|
||||
self.k_conv1d.bias,
|
||||
activation="silu",
|
||||
conv_states=conv_state_k,
|
||||
has_initial_state=has_initial_state,
|
||||
cache_indices=non_spec_state_indices_tensor,
|
||||
query_start_loc=non_spec_query_start_loc,
|
||||
metadata=attn_metadata_narrowed,
|
||||
).transpose(0, 1)
|
||||
v = causal_conv1d_fn(
|
||||
v_proj_states,
|
||||
v_conv_weights,
|
||||
self.v_conv1d.bias,
|
||||
activation="silu",
|
||||
conv_states=conv_state_v,
|
||||
has_initial_state=has_initial_state,
|
||||
cache_indices=non_spec_state_indices_tensor,
|
||||
query_start_loc=non_spec_query_start_loc,
|
||||
metadata=attn_metadata_narrowed,
|
||||
).transpose(0, 1)
|
||||
# Split tokens into the multi-query spec-decode part and the remaining
|
||||
# (prefill / plain decode) part.
|
||||
if spec_sequence_masks is not None:
|
||||
if m.num_prefills == 0 and m.num_decodes == 0:
|
||||
mixed_qkv_spec = mixed_qkv
|
||||
g1_spec, beta_spec = g1, beta
|
||||
mixed_qkv_ns = g1_ns = beta_ns = None
|
||||
else:
|
||||
mixed_qkv_spec = mixed_qkv.index_select(0, spec_token_indx)
|
||||
g1_spec = g1.index_select(1, spec_token_indx)
|
||||
beta_spec = beta.index_select(1, spec_token_indx)
|
||||
mixed_qkv_ns = mixed_qkv.index_select(0, non_spec_token_indx)
|
||||
g1_ns = g1.index_select(1, non_spec_token_indx)
|
||||
beta_ns = beta.index_select(1, non_spec_token_indx)
|
||||
else:
|
||||
assert non_spec_state_indices_tensor is not None
|
||||
decode_conv_indices = non_spec_state_indices_tensor[
|
||||
: attn_metadata_narrowed.num_actual_tokens
|
||||
]
|
||||
q = causal_conv1d_update(
|
||||
q_proj_states,
|
||||
conv_state_q,
|
||||
q_conv_weights,
|
||||
self.q_conv1d.bias,
|
||||
activation="silu",
|
||||
conv_state_indices=decode_conv_indices,
|
||||
validate_data=True,
|
||||
)
|
||||
k = causal_conv1d_update(
|
||||
k_proj_states,
|
||||
conv_state_k,
|
||||
k_conv_weights,
|
||||
self.k_conv1d.bias,
|
||||
activation="silu",
|
||||
conv_state_indices=decode_conv_indices,
|
||||
validate_data=True,
|
||||
)
|
||||
v = causal_conv1d_update(
|
||||
v_proj_states,
|
||||
conv_state_v,
|
||||
v_conv_weights,
|
||||
self.v_conv1d.bias,
|
||||
activation="silu",
|
||||
conv_state_indices=decode_conv_indices,
|
||||
validate_data=True,
|
||||
)
|
||||
mixed_qkv_spec = g1_spec = beta_spec = None
|
||||
mixed_qkv_ns, g1_ns, beta_ns = mixed_qkv, g1, beta
|
||||
|
||||
q, k, v = map(
|
||||
lambda x: rearrange(x, "n (h d) -> 1 n h d", d=self.head_dim), (q, k, v)
|
||||
)
|
||||
# ---------- spec-decode multi-query path ----------
|
||||
core_attn_out_spec = None
|
||||
if spec_sequence_masks is not None:
|
||||
assert spec_state_indices_tensor is not None
|
||||
assert spec_query_start_loc is not None
|
||||
spec_conv_indices = spec_state_indices_tensor[:, 0][: m.num_spec_decodes]
|
||||
spec_max_query_len = spec_state_indices_tensor.size(-1)
|
||||
|
||||
if attn_metadata_narrowed.num_prefills > 0:
|
||||
assert non_spec_state_indices_tensor is not None
|
||||
assert has_initial_state is not None
|
||||
zero_idx = non_spec_state_indices_tensor[~has_initial_state]
|
||||
recurrent_state[zero_idx] = 0
|
||||
initial_state = recurrent_state[non_spec_state_indices_tensor].contiguous()
|
||||
(
|
||||
core_attn_out_non_spec,
|
||||
last_recurrent_state,
|
||||
) = chunk_kda_with_fused_gate(
|
||||
q=q,
|
||||
k=k,
|
||||
v=v,
|
||||
raw_g=g1,
|
||||
beta=beta,
|
||||
# Sibling beta and, for full-rank gates, output-gate views remain
|
||||
# live, so write the convolution output separately.
|
||||
spec_conv_out = torch.empty(
|
||||
mixed_qkv_spec.shape,
|
||||
dtype=mixed_qkv_spec.dtype,
|
||||
device=mixed_qkv_spec.device,
|
||||
)
|
||||
mixed_qkv_spec = causal_conv1d_update(
|
||||
mixed_qkv_spec,
|
||||
conv_state,
|
||||
conv_weights,
|
||||
self.conv1d.bias,
|
||||
activation="silu",
|
||||
conv_state_indices=spec_conv_indices,
|
||||
num_accepted_tokens=num_accepted_tokens,
|
||||
query_start_loc=spec_query_start_loc,
|
||||
max_query_len=spec_max_query_len,
|
||||
validate_data=False,
|
||||
out=spec_conv_out,
|
||||
)
|
||||
q_spec, k_spec, v_spec = (
|
||||
rearrange(x, "n (h d) -> 1 n h d", d=self.head_dim)
|
||||
for x in mixed_qkv_spec.split(self.local_projection_size, dim=-1)
|
||||
)
|
||||
spec_cu_seqlens = spec_query_start_loc[: m.num_spec_decodes + 1]
|
||||
# Spec-only batches write directly into core_attn_out.
|
||||
spec_out = (
|
||||
core_attn_out[:, : q_spec.shape[1]]
|
||||
if m.num_prefills == 0 and m.num_decodes == 0
|
||||
else None
|
||||
)
|
||||
core_attn_out_spec, _ = fused_recurrent_kda(
|
||||
q=q_spec,
|
||||
k=k_spec,
|
||||
v=v_spec,
|
||||
raw_g=g1_spec,
|
||||
raw_beta=beta_spec,
|
||||
A_log=self.A_log,
|
||||
g_bias=self.dt_bias,
|
||||
initial_state=initial_state,
|
||||
output_final_state=True,
|
||||
use_qk_l2norm_in_kernel=True,
|
||||
cu_seqlens=non_spec_query_start_loc,
|
||||
)
|
||||
# Init cache
|
||||
recurrent_state[non_spec_state_indices_tensor] = last_recurrent_state
|
||||
else:
|
||||
assert non_spec_query_start_loc is not None
|
||||
g1 = fused_kda_gate(
|
||||
rearrange(g1, "1 n h d -> n (h d)"),
|
||||
self.A_log,
|
||||
self.head_dim,
|
||||
g_bias=self.dt_bias,
|
||||
).unsqueeze(0)
|
||||
(
|
||||
core_attn_out_non_spec,
|
||||
last_recurrent_state,
|
||||
) = fused_recurrent_kda(
|
||||
q=q,
|
||||
k=k,
|
||||
v=v,
|
||||
g=g1,
|
||||
beta=beta,
|
||||
dt_bias=self.dt_bias,
|
||||
lower_bound=self.gate_lower_bound,
|
||||
initial_state=recurrent_state,
|
||||
use_qk_l2norm_in_kernel=True,
|
||||
cu_seqlens=non_spec_query_start_loc[
|
||||
: attn_metadata_narrowed.num_decodes + 1
|
||||
],
|
||||
ssm_state_indices=non_spec_state_indices_tensor,
|
||||
cu_seqlens=spec_cu_seqlens,
|
||||
ssm_state_indices=spec_state_indices_tensor,
|
||||
num_accepted_tokens=num_accepted_tokens,
|
||||
out=spec_out,
|
||||
)
|
||||
core_attn_out[0, :num_actual_tokens] = core_attn_out_non_spec[
|
||||
0, :num_actual_tokens
|
||||
]
|
||||
|
||||
# ---------- non-spec path (prefill or plain decode) ----------
|
||||
core_attn_out_non_spec = None
|
||||
if mixed_qkv_ns is not None:
|
||||
assert g1_ns is not None and beta_ns is not None
|
||||
if m.num_prefills > 0:
|
||||
q_ns, k_ns, v_ns = mixed_qkv_ns.split(
|
||||
self.local_projection_size, dim=-1
|
||||
)
|
||||
|
||||
# Packed prefill conv would require copying V solely to make
|
||||
# it dense for KDA. Separate calls accept the strided inputs
|
||||
# and produce dense Q/K/V without that extra traffic.
|
||||
# TODO: Use packed conv once every KDA prefill backend accepts
|
||||
# row-strided Q/K/V directly.
|
||||
def _prefill_conv(
|
||||
x: torch.Tensor,
|
||||
state: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
return causal_conv1d_fn(
|
||||
x.transpose(0, 1),
|
||||
weight,
|
||||
None,
|
||||
activation="silu",
|
||||
conv_states=state,
|
||||
has_initial_state=has_initial_state,
|
||||
cache_indices=non_spec_state_indices_tensor,
|
||||
query_start_loc=non_spec_query_start_loc,
|
||||
metadata=m,
|
||||
).transpose(0, 1)
|
||||
|
||||
q_ns = _prefill_conv(q_ns, q_conv_state, q_conv_weight)
|
||||
k_ns = _prefill_conv(k_ns, k_conv_state, k_conv_weight)
|
||||
v_ns = _prefill_conv(v_ns, v_conv_state, v_conv_weight)
|
||||
q_ns, k_ns, v_ns = (
|
||||
rearrange(x, "n (h d) -> 1 n h d", d=self.head_dim)
|
||||
for x in (q_ns, k_ns, v_ns)
|
||||
)
|
||||
|
||||
assert non_spec_state_indices_tensor is not None
|
||||
assert has_initial_state is not None
|
||||
initial_state = gather_initial_states(
|
||||
recurrent_state,
|
||||
non_spec_state_indices_tensor,
|
||||
has_initial_state,
|
||||
)
|
||||
(
|
||||
core_attn_out_non_spec,
|
||||
last_recurrent_state,
|
||||
) = chunk_kda_with_fused_gate(
|
||||
q=q_ns,
|
||||
k=k_ns,
|
||||
v=v_ns,
|
||||
raw_g=g1_ns,
|
||||
raw_beta=beta_ns,
|
||||
A_log=self.A_log,
|
||||
g_bias=self.dt_bias,
|
||||
lower_bound=self.gate_lower_bound,
|
||||
initial_state=initial_state,
|
||||
output_final_state=True,
|
||||
use_qk_l2norm_in_kernel=True,
|
||||
cu_seqlens=non_spec_query_start_loc,
|
||||
)
|
||||
# Init cache
|
||||
recurrent_state[non_spec_state_indices_tensor] = last_recurrent_state
|
||||
|
||||
else:
|
||||
# pure-decode non-spec batch
|
||||
assert non_spec_state_indices_tensor is not None
|
||||
decode_conv_indices = non_spec_state_indices_tensor[
|
||||
: mixed_qkv_ns.size(0)
|
||||
]
|
||||
# Sibling beta and, for full-rank gates, output-gate views
|
||||
# remain live, so write the conv output separately.
|
||||
packed_conv_out = torch.empty(
|
||||
mixed_qkv_ns.shape,
|
||||
dtype=mixed_qkv_ns.dtype,
|
||||
device=mixed_qkv_ns.device,
|
||||
)
|
||||
mixed_qkv_ns = causal_conv1d_update(
|
||||
mixed_qkv_ns,
|
||||
conv_state,
|
||||
conv_weights,
|
||||
self.conv1d.bias,
|
||||
activation="silu",
|
||||
conv_state_indices=decode_conv_indices,
|
||||
validate_data=True,
|
||||
out=packed_conv_out,
|
||||
)
|
||||
core_attn_out_non_spec, _ = fused_recurrent_kda_packed_decode(
|
||||
mixed_qkv=mixed_qkv_ns,
|
||||
raw_g=g1_ns,
|
||||
raw_beta=beta_ns,
|
||||
A_log=self.A_log,
|
||||
dt_bias=self.dt_bias,
|
||||
lower_bound=self.gate_lower_bound,
|
||||
initial_state=recurrent_state,
|
||||
state_indices=decode_conv_indices,
|
||||
)
|
||||
|
||||
# ---------- merge spec and non-spec outputs ----------
|
||||
if core_attn_out_spec is not None and core_attn_out_non_spec is not None:
|
||||
# Mixed batches require indexed placement in the original order.
|
||||
merged = torch.empty(
|
||||
(1, num_actual_tokens, *core_attn_out_spec.shape[2:]),
|
||||
dtype=core_attn_out_spec.dtype,
|
||||
device=core_attn_out_spec.device,
|
||||
)
|
||||
merged.index_copy_(1, spec_token_indx, core_attn_out_spec)
|
||||
merged.index_copy_(1, non_spec_token_indx, core_attn_out_non_spec)
|
||||
core_attn_out[0, :num_actual_tokens] = merged[0, :num_actual_tokens]
|
||||
elif core_attn_out_non_spec is not None:
|
||||
core_attn_out[0, :num_actual_tokens] = core_attn_out_non_spec[
|
||||
0, :num_actual_tokens
|
||||
]
|
||||
else:
|
||||
assert core_attn_out_spec is not None
|
||||
core_attn_out.copy_(self.o_norm(core_attn_out, g2))
|
||||
|
||||
@@ -288,7 +288,7 @@ class MambaStateShapeCalculator:
|
||||
|
||||
conv_dim = proj_size + 2 * proj_k_size
|
||||
conv_state_shape = cls._orient_conv_shape(
|
||||
divide(conv_dim, tp_world_size), conv_kernel_size - 1
|
||||
divide(conv_dim, tp_world_size), conv_kernel_size - 1 + num_spec
|
||||
)
|
||||
recurrent_state_shape = (divide(num_heads, tp_world_size), head_dim, head_dim)
|
||||
return (conv_state_shape, recurrent_state_shape)
|
||||
|
||||
@@ -8,11 +8,12 @@
|
||||
import numpy as np
|
||||
import torch
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.v1.attention.backends.utils import NULL_BLOCK_ID, PAD_SLOT_ID
|
||||
|
||||
|
||||
@triton.jit()
|
||||
@triton.jit(do_not_specialize_on_alignment=["num_cache_lines"])
|
||||
def _causal_conv1d_fwd_kernel( # continuous batching
|
||||
# Pointers to matrices
|
||||
x_ptr, # (dim, cu_seqlen) holding `batch` of actual sequences + padded sequences
|
||||
@@ -33,7 +34,7 @@ def _causal_conv1d_fwd_kernel( # continuous batching
|
||||
o_ptr, # (dim, seqlen) - actually pointing to x_ptr
|
||||
# Matrix dimensions
|
||||
dim: tl.constexpr,
|
||||
num_cache_lines: tl.constexpr, # added to support vLLM larger cache lines
|
||||
num_cache_lines, # added to support vLLM larger cache lines
|
||||
# Strides
|
||||
stride_x_dim: tl.constexpr, # stride to get to next feature-value,
|
||||
stride_x_token: tl.int64, # stride to get to next token (same feature-index, same sequence-index)
|
||||
@@ -58,6 +59,7 @@ def _causal_conv1d_fwd_kernel( # continuous batching
|
||||
NP2_STATELEN: tl.constexpr,
|
||||
BLOCK_M: tl.constexpr,
|
||||
BLOCK_N: tl.constexpr,
|
||||
launch_pdl: tl.constexpr,
|
||||
):
|
||||
conv_states_ptr = initial_states_ptr
|
||||
conv_state_indices_ptr = cache_indices_ptr
|
||||
@@ -68,6 +70,9 @@ def _causal_conv1d_fwd_kernel( # continuous batching
|
||||
KERNEL_WIDTH - 1
|
||||
) # can be passed via argument if it's not the same as this value
|
||||
|
||||
if launch_pdl:
|
||||
tl.extra.cuda.gdc_wait()
|
||||
|
||||
# one program handles one chunk in a single sequence
|
||||
# rather than mixing sequences - to make updating initial_states across sequences efficiently
|
||||
|
||||
@@ -79,6 +84,8 @@ def _causal_conv1d_fwd_kernel( # continuous batching
|
||||
idx_feats = tl.program_id(1) * BLOCK_N + tl.arange(0, BLOCK_N)
|
||||
|
||||
if idx_seq == pad_slot_id:
|
||||
if launch_pdl:
|
||||
tl.extra.cuda.gdc_launch_dependents()
|
||||
return
|
||||
|
||||
sequence_start_index = tl.load(query_start_loc_ptr + idx_seq)
|
||||
@@ -136,6 +143,8 @@ def _causal_conv1d_fwd_kernel( # continuous batching
|
||||
if HAS_NULL_BLOCK: # noqa
|
||||
if conv_states_input_coord == null_block_id:
|
||||
# not processing as this is a null block (padding)
|
||||
if launch_pdl:
|
||||
tl.extra.cuda.gdc_launch_dependents()
|
||||
return
|
||||
conv_states_base = (
|
||||
conv_states_ptr
|
||||
@@ -408,6 +417,10 @@ def _causal_conv1d_fwd_kernel( # continuous batching
|
||||
w_ptrs = w_base + (3 * stride_w_width) # [BLOCK_N] tensor
|
||||
w_col3 = tl.load(w_ptrs, mask_w, other=0.0)
|
||||
mask_x_1d = idx_feats < dim
|
||||
|
||||
if launch_pdl:
|
||||
tl.extra.cuda.gdc_launch_dependents()
|
||||
|
||||
for idx_token in range(segment_len):
|
||||
acc = acc_preload
|
||||
|
||||
@@ -741,11 +754,12 @@ def causal_conv1d_fn(
|
||||
BLOCK_M=BLOCK_M,
|
||||
BLOCK_N=256,
|
||||
num_stages=2,
|
||||
launch_pdl=current_platform.is_arch_support_pdl(),
|
||||
)
|
||||
return out.to(original_x_dtype)
|
||||
|
||||
|
||||
@triton.jit()
|
||||
@triton.jit(do_not_specialize_on_alignment=["num_cache_lines"])
|
||||
def _causal_conv1d_update_kernel(
|
||||
# Pointers to matrices
|
||||
x_ptr, # (batch, dim, seqlen)
|
||||
@@ -763,7 +777,7 @@ def _causal_conv1d_update_kernel(
|
||||
dim: tl.constexpr,
|
||||
seqlen: tl.constexpr,
|
||||
state_len: tl.constexpr,
|
||||
num_cache_lines: tl.constexpr, # added to support vLLM larger cache lines
|
||||
num_cache_lines, # added to support vLLM larger cache lines
|
||||
# Strides
|
||||
stride_x_seq: tl.constexpr,
|
||||
stride_x_dim: tl.constexpr,
|
||||
@@ -789,10 +803,16 @@ def _causal_conv1d_update_kernel(
|
||||
NP2_STATELEN: tl.constexpr,
|
||||
HAS_NULL_BLOCK: tl.constexpr,
|
||||
BLOCK_N: tl.constexpr,
|
||||
launch_pdl: tl.constexpr,
|
||||
):
|
||||
if launch_pdl:
|
||||
tl.extra.cuda.gdc_wait()
|
||||
|
||||
# ruff: noqa: E501
|
||||
idx_seq = tl.program_id(0)
|
||||
if idx_seq >= batch:
|
||||
if launch_pdl:
|
||||
tl.extra.cuda.gdc_launch_dependents()
|
||||
return
|
||||
|
||||
# [BLOCK_N,] elements along the feature-dimension (channel)
|
||||
@@ -814,6 +834,8 @@ def _causal_conv1d_update_kernel(
|
||||
if HAS_NULL_BLOCK: # noqa
|
||||
if conv_states_input_coord == null_block_id:
|
||||
# not processing as this is not the actual sequence
|
||||
if launch_pdl:
|
||||
tl.extra.cuda.gdc_launch_dependents()
|
||||
return
|
||||
|
||||
if IS_VARLEN:
|
||||
@@ -831,6 +853,8 @@ def _causal_conv1d_update_kernel(
|
||||
o_offset = idx_seq * stride_o_seq
|
||||
|
||||
if query_start_index == query_end_index:
|
||||
if launch_pdl:
|
||||
tl.extra.cuda.gdc_launch_dependents()
|
||||
return
|
||||
|
||||
if IS_SPEC_DECODING:
|
||||
@@ -969,6 +993,9 @@ def _causal_conv1d_update_kernel(
|
||||
mask_x_1d = idx_feats < dim
|
||||
|
||||
# STEP 5: compute each token
|
||||
if launch_pdl:
|
||||
tl.extra.cuda.gdc_launch_dependents()
|
||||
|
||||
for idx_token in tl.range(seqlen):
|
||||
acc = acc_preload
|
||||
|
||||
@@ -1245,14 +1272,13 @@ def causal_conv1d_update(
|
||||
NP2_STATELEN=np2_statelen,
|
||||
HAS_NULL_BLOCK=null_block_id is not None,
|
||||
BLOCK_N=256,
|
||||
launch_pdl=current_platform.is_arch_support_pdl(),
|
||||
)
|
||||
if unsqueeze:
|
||||
out = out.squeeze(-1)
|
||||
return out.to(original_x_dtype)
|
||||
|
||||
|
||||
from vllm.platforms import current_platform # noqa: E402
|
||||
|
||||
if current_platform.is_cpu():
|
||||
from vllm.model_executor.layers.mamba.ops.cpu.causal_conv1d import (
|
||||
causal_conv1d_fn_cpu,
|
||||
|
||||
@@ -28,6 +28,7 @@ class MLAModules:
|
||||
is_sparse: bool
|
||||
topk_indices_buffer: torch.Tensor | None
|
||||
indexer_rotary_emb: torch.nn.Module | None = None
|
||||
g_proj: torch.nn.Module | None = None
|
||||
|
||||
|
||||
# --8<-- [start:multi_head_latent_attention]
|
||||
@@ -66,6 +67,7 @@ class MultiHeadLatentAttentionWrapper(PluggableLayer):
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
prefix: str = "",
|
||||
skip_topk: bool = False,
|
||||
non_causal_multi_token_decode: bool = False,
|
||||
allow_short_prefill_indexer_scoring_skip: bool = False,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
@@ -89,6 +91,7 @@ class MultiHeadLatentAttentionWrapper(PluggableLayer):
|
||||
self.indexer = mla_modules.indexer
|
||||
self.indexer_rope_emb = mla_modules.indexer_rotary_emb
|
||||
self.is_sparse = mla_modules.is_sparse
|
||||
self.g_proj = mla_modules.g_proj
|
||||
|
||||
# Whether to skip top-k token selection computation in this layer.
|
||||
# When True, the indexer will not be called, and the layer will reuse
|
||||
@@ -120,6 +123,7 @@ class MultiHeadLatentAttentionWrapper(PluggableLayer):
|
||||
use_sparse=self.is_sparse,
|
||||
indexer=self.indexer,
|
||||
topk_indices_buffer=mla_modules.topk_indices_buffer,
|
||||
non_causal_multi_token_decode=non_causal_multi_token_decode,
|
||||
)
|
||||
indexer_op = getattr(self.indexer, "indexer_op", None)
|
||||
if indexer_op is not None and hasattr(
|
||||
@@ -216,4 +220,7 @@ class MultiHeadLatentAttentionWrapper(PluggableLayer):
|
||||
q_dcp_replicated=q_dcp_replicated,
|
||||
)
|
||||
|
||||
if self.g_proj is not None:
|
||||
attn_out = attn_out * self.g_proj(hidden_states)[0].sigmoid()
|
||||
|
||||
return self.o_proj(attn_out)[0]
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
|
||||
import torch
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.attention import Attention
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
@@ -17,6 +18,7 @@ from vllm.model_executor.layers.fused_moe import modular_kernel as mk
|
||||
from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import (
|
||||
TRITON_BACKENDS,
|
||||
Mxfp4MoeBackend,
|
||||
backend_to_kernel_cls,
|
||||
convert_gpt_oss_weight_to_mxfp4_moe_kernel_format,
|
||||
convert_weight_to_mxfp4_moe_kernel_format,
|
||||
make_mxfp4_moe_kernel,
|
||||
@@ -68,8 +70,11 @@ class Mxfp4Config(QuantizationConfig):
|
||||
def get_config_filenames(cls) -> list[str]:
|
||||
return []
|
||||
|
||||
# TODO (zyongye) This is only temporaty fallback.
|
||||
# We should have `Mxfp4MoEMethod` after this migration is complete.
|
||||
def _make_moe_method(self, moe: FusedMoEConfig) -> FusedMoEMethodBase:
|
||||
"""MoE method for RoutedExperts. Subclasses override to pick a
|
||||
checkpoint-specific kernel family."""
|
||||
return Mxfp4MoEMethod(moe)
|
||||
|
||||
def get_quant_method(
|
||||
self, layer: torch.nn.Module, prefix: str
|
||||
) -> "QuantizeMethodBase | None":
|
||||
@@ -86,7 +91,7 @@ class Mxfp4Config(QuantizationConfig):
|
||||
)
|
||||
return UnquantizedLinearMethod()
|
||||
elif isinstance(layer, RoutedExperts):
|
||||
return GptOssMxfp4MoEMethod(layer.moe_config)
|
||||
return self._make_moe_method(layer.moe_config)
|
||||
elif isinstance(layer, Attention):
|
||||
logger.debug_once(
|
||||
"MXFP4 attention layer is not implemented. "
|
||||
@@ -131,6 +136,9 @@ class GptOssMxfp4Config(Mxfp4Config):
|
||||
return None
|
||||
return "gpt_oss_mxfp4"
|
||||
|
||||
def _make_moe_method(self, moe: FusedMoEConfig) -> FusedMoEMethodBase:
|
||||
return GptOssMxfp4MoEMethod(moe)
|
||||
|
||||
|
||||
class GptOssMxfp4MoEMethod(FusedMoEMethodBase):
|
||||
"""MXFP4 MoE quantization method."""
|
||||
@@ -475,13 +483,45 @@ class GptOssMxfp4MoEMethod(FusedMoEMethodBase):
|
||||
)
|
||||
|
||||
|
||||
def _use_k3_situ_aiter(moe: FusedMoEConfig) -> bool:
|
||||
"""Whether Kimi-K3's SiTU MXFP4 MoE should use the AITER A16W4 kernel.
|
||||
|
||||
K3 is weight-only MXFP4 (W4A16) with SiTU activation, which the generic
|
||||
MXFP4 backend selector does not cover; route it to AITER on gfx950.
|
||||
"""
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
if not current_platform.is_rocm():
|
||||
return False
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
from vllm.platforms.rocm import on_gfx950
|
||||
|
||||
return (
|
||||
rocm_aiter_ops.is_fused_moe_enabled()
|
||||
and on_gfx950()
|
||||
and moe.activation == MoEActivation.SITU
|
||||
and moe.activation_situ_linear_beta is not None
|
||||
and rocm_aiter_ops.get_aiter_activation_type("situ") is not None
|
||||
)
|
||||
|
||||
|
||||
class Mxfp4MoEMethod(FusedMoEMethodBase):
|
||||
"""MXFP4 MoE quantization method."""
|
||||
|
||||
def __init__(self, moe: FusedMoEConfig):
|
||||
super().__init__(moe)
|
||||
self.weight_dtype = "mxfp4"
|
||||
self.mxfp4_backend, self.experts_cls = select_deepseek_v4_mxfp4_moe_backend(moe)
|
||||
self.is_k3_situ_aiter = _use_k3_situ_aiter(moe)
|
||||
self.experts_cls: type[mk.FusedMoEExperts] | None
|
||||
if self.is_k3_situ_aiter:
|
||||
self.mxfp4_backend = Mxfp4MoeBackend.AITER_MXFP4_BF16
|
||||
self.experts_cls = backend_to_kernel_cls(self.mxfp4_backend)[0]
|
||||
logger.info_once("Using AITER_MXFP4_BF16 for Kimi-K3 SiTU MXFP4 MoE.")
|
||||
else:
|
||||
self.mxfp4_backend, self.experts_cls = select_deepseek_v4_mxfp4_moe_backend(
|
||||
moe
|
||||
)
|
||||
|
||||
self.max_capture_size = moe.max_capture_size
|
||||
|
||||
@@ -523,6 +563,11 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
|
||||
act_dtype=act_dtype,
|
||||
moe_parallel_config=moe_parallel_config,
|
||||
)
|
||||
if self.is_k3_situ_aiter:
|
||||
# K3's AITER A16W4 kernel handles K3's native intermediate size
|
||||
# (moe_intermediate 3072; e.g. 384/partition at TP8); the generic
|
||||
# 256 round-up would inflate weights and OOM.
|
||||
return hidden_size, intermediate_size_per_partition
|
||||
return mxfp4_round_up_hidden_size_and_intermediate_size(
|
||||
self.mxfp4_backend, hidden_size, intermediate_size_per_partition
|
||||
)
|
||||
@@ -724,7 +769,58 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
|
||||
layer=layer,
|
||||
)
|
||||
|
||||
def _setup_kernel_k3_situ(self, layer: RoutedExperts) -> None:
|
||||
# K3's AITER A16W4 kernel wants the separated ([gate_all, up_all])
|
||||
# stage-1 layout, unlike the interleaved gpt-oss/DeepSeek path in
|
||||
# convert_weight_to_mxfp4_moe_kernel_format. Preshuffle once here.
|
||||
from aiter.utility.fp4_utils import e8m0_shuffle
|
||||
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
|
||||
fp4_dtype = torch.float4_e2m1fn_x2
|
||||
e8m0_dtype = torch.float8_e8m0fnu
|
||||
num_experts = layer.w13_weight.shape[0]
|
||||
|
||||
# a8w4 (AITER_SITUV2_A8W4=1) uses the gate/up-interleaved (_gui_) fp8
|
||||
# flydsl kernels, which need w13 weight+scale in interleave layout.
|
||||
# Default a16w4 keeps the separated layout.
|
||||
guinterleave = envs.AITER_SITUV2_A8W4
|
||||
w13 = rocm_aiter_ops.shuffle_weight_a16w4(
|
||||
layer.w13_weight.data.view(fp4_dtype), 16, guinterleave
|
||||
)
|
||||
w2 = rocm_aiter_ops.shuffle_weight_a16w4(
|
||||
layer.w2_weight.data.view(fp4_dtype), 16, False
|
||||
)
|
||||
w13_scale_raw = layer.w13_weight_scale.data.view(e8m0_dtype)
|
||||
w2_scale_raw = layer.w2_weight_scale.data.view(e8m0_dtype)
|
||||
w13_scale = rocm_aiter_ops.shuffle_scale_a16w4(
|
||||
w13_scale_raw.view(-1, w13_scale_raw.shape[-1]), num_experts, guinterleave
|
||||
)
|
||||
w2_scale = e8m0_shuffle(w2_scale_raw.view(-1, w2_scale_raw.shape[-1]))
|
||||
|
||||
replace_parameter(layer, "w13_weight", w13)
|
||||
replace_parameter(layer, "w2_weight", w2)
|
||||
replace_parameter(layer, "w13_weight_scale", w13_scale)
|
||||
replace_parameter(layer, "w2_weight_scale", w2_scale)
|
||||
layer.w13_weight.is_shuffled = True
|
||||
layer.w2_weight.is_shuffled = True
|
||||
|
||||
self.moe_quant_config = self.get_fused_moe_quant_config(layer)
|
||||
if self.moe_quant_config is not None and self.experts_cls is not None:
|
||||
self.moe_kernel = make_mxfp4_moe_kernel(
|
||||
moe_quant_config=self.moe_quant_config,
|
||||
moe_config=self.moe,
|
||||
mxfp4_backend=self.mxfp4_backend,
|
||||
experts_cls=self.experts_cls,
|
||||
routing_tables=layer._expert_routing_tables(),
|
||||
layer=layer,
|
||||
)
|
||||
|
||||
def process_weights_after_loading(self, layer):
|
||||
if self.is_k3_situ_aiter:
|
||||
self._setup_kernel_k3_situ(layer)
|
||||
return
|
||||
|
||||
w13 = layer.w13_weight
|
||||
w2 = layer.w2_weight
|
||||
w13_scale = layer.w13_weight_scale
|
||||
@@ -819,4 +915,8 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
|
||||
global_num_experts=layer.global_num_experts,
|
||||
expert_map=layer.expert_map,
|
||||
apply_router_weight_on_input=layer.apply_router_weight_on_input,
|
||||
num_expert_group=layer.num_expert_group,
|
||||
topk_group=layer.topk_group,
|
||||
e_score_correction_bias=layer.e_score_correction_bias,
|
||||
routed_scaling_factor=layer.routed_scaling_factor,
|
||||
)
|
||||
|
||||
@@ -18,20 +18,34 @@ def activation_to_flashinfer_int(activation: MoEActivation) -> int:
|
||||
return activation_to_flashinfer_type(activation).value
|
||||
|
||||
|
||||
def has_flashinfer_situ_activation() -> bool:
|
||||
try:
|
||||
from flashinfer.fused_moe.core import ActivationType
|
||||
except ImportError:
|
||||
return False
|
||||
return hasattr(ActivationType, "Situ")
|
||||
|
||||
|
||||
def activation_to_flashinfer_type(activation: MoEActivation) -> "ActivationType":
|
||||
from flashinfer.fused_moe.core import ActivationType
|
||||
|
||||
if activation == MoEActivation.SITU:
|
||||
situ = getattr(ActivationType, "Situ", None)
|
||||
if situ is None:
|
||||
raise ValueError("The installed FlashInfer does not support SITU")
|
||||
return situ
|
||||
|
||||
# silu and gelu are mapped to their gated versions SwiGLU and GeGLU respectively
|
||||
ACTIVATION_TO_FI_ACTIVATION = {
|
||||
MoEActivation.SILU_NO_MUL: ActivationType.Silu,
|
||||
MoEActivation.GELU_NO_MUL: ActivationType.Gelu,
|
||||
MoEActivation.SILU: ActivationType.Swiglu,
|
||||
# SwiGLU-OAI uses Swiglu; the OAI alpha/beta/clamp come from gemm1_* args.
|
||||
MoEActivation.SWIGLUOAI: ActivationType.Swiglu,
|
||||
MoEActivation.SWIGLUOAI_UNINTERLEAVE: ActivationType.Swiglu,
|
||||
MoEActivation.GELU: ActivationType.Geglu,
|
||||
MoEActivation.GELU_TANH: ActivationType.Geglu,
|
||||
MoEActivation.RELU2_NO_MUL: ActivationType.Relu2,
|
||||
MoEActivation.SWIGLUOAI_UNINTERLEAVE: ActivationType.Swiglu,
|
||||
}
|
||||
return ACTIVATION_TO_FI_ACTIVATION[activation]
|
||||
|
||||
|
||||
@@ -474,7 +474,11 @@ def prepare_moe_fp4_layer_for_marlin(
|
||||
|
||||
group_size = 16 if is_nvfp4 else 32
|
||||
|
||||
e = layer.moe_config.num_experts
|
||||
# Use the per-rank (local) expert count: under expert parallelism the
|
||||
# w13_weight/w2_weight tensors only hold this rank's experts (created with
|
||||
# local_num_experts), whereas moe_config.num_experts is the global count.
|
||||
# With no EP the two are equal, so the non-EP path is unchanged.
|
||||
e = layer.moe_config.num_local_experts
|
||||
k = layer.moe_config.hidden_dim
|
||||
n = layer.moe_config.intermediate_size_per_partition
|
||||
|
||||
|
||||
@@ -41,8 +41,8 @@ def _mxfp8_e4m3_quantize_torch(
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Naive MXFP8 quantization.
|
||||
For each block of 32 elements along the last dimension, compute a
|
||||
shared e8m0 scale (the biased exponent of the block-wise amax)
|
||||
and quantize each element to float8_e4m3fn.
|
||||
shared e8m0 scale that fits the block-wise amax into the finite
|
||||
float8_e4m3fn range, and quantize each element to float8_e4m3fn.
|
||||
|
||||
Returns (quantized_values [same shape, fp8], scales uint8).
|
||||
Scale shape depends on is_sf_swizzled_layout:
|
||||
@@ -58,7 +58,8 @@ def _mxfp8_e4m3_quantize_torch(
|
||||
|
||||
amax = x_blocked.abs().amax(dim=-1)
|
||||
amax = amax.clamp(min=torch.finfo(torch.float32).tiny)
|
||||
scale_biased = torch.floor(torch.log2(amax)) + 127.0
|
||||
fp8_max = torch.finfo(MXFP8_VALUE_DTYPE).max
|
||||
scale_biased = torch.ceil(torch.log2(amax / fp8_max)) + 127.0
|
||||
scale_biased = scale_biased.clamp(0, 254)
|
||||
scales_uint8 = scale_biased.to(torch.uint8)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user