[New model] Kimi K3 (#50000)
Signed-off-by: zjy0516 <riverclouds.zhu@qq.com> Signed-off-by: Isotr0py <Isotr0py@outlook.com> Signed-off-by: Bugen Zhao <i@bugenzhao.com> Signed-off-by: Jee Jee Li <jeejeelee@inferact.ai> Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai> Signed-off-by: Yongye Zhu <zyy1102000@gmail.com> Signed-off-by: wzhao18 <wzhao18.sz@gmail.com> Co-authored-by: Isotr0py <Isotr0py@outlook.com> Co-authored-by: Thien Tran <gau.nernst@yahoo.com.sg> Co-authored-by: Bugen Zhao <i@bugenzhao.com> Co-authored-by: Yongye Zhu <zyy1102000@gmail.com> Co-authored-by: Ziming Huang <zelda.huanghuang@gmail.com> Co-authored-by: Roger Wang <hey@rogerw.io> Co-authored-by: Isotr0py <mozf@inferact.ai> Co-authored-by: aoshen02 <aoshen02@users.noreply.github.com> Co-authored-by: Woosuk Kwon <woosuk@inferact.ai> Co-authored-by: Jee Jee Li <jeejeelee@inferact.ai> Co-authored-by: aoshen02 <aoshen@inferact.ai> Co-authored-by: Summer Yang <girasoleyang@gmail.com> Co-authored-by: Kevin H. Luu <khluu000@gmail.com> Co-authored-by: Bowen Wang <abmfy@icloud.com> Co-authored-by: gnovack <novackgm@gmail.com> Co-authored-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai> Co-authored-by: xiaozhoupy <peiyuanzhou1994@gmail.com> Co-authored-by: Roy Wang <yasong.wang@inferact.ai> Co-authored-by: Jeff (Junze) Ma <93145857+majunze2001@users.noreply.github.com> Co-authored-by: Codex <codex@openai.com> Co-authored-by: Wei Zhao <51183510+wzhao18@users.noreply.github.com>
This commit is contained in:
co-authored by
Isotr0py
Thien Tran
Bugen Zhao
Yongye Zhu
Ziming Huang
Roger Wang
Isotr0py
aoshen02
Woosuk Kwon
Jee Jee Li
aoshen02
Summer Yang
Kevin H. Luu
Bowen Wang
gnovack
Nick Hill
Yifan Qiao
xiaozhoupy
Roy Wang
Jeff Ma
Codex
Wei Zhao
parent
0c64be8873
commit
aeeb36b1f1
@@ -272,6 +272,7 @@ from vllm.v1.attention.backends.mla.prefill import (
|
||||
)
|
||||
from vllm.v1.attention.backends.utils import (
|
||||
get_dcp_local_seq_lens,
|
||||
get_num_attention_heads_from_layers,
|
||||
split_decodes_and_prefills,
|
||||
)
|
||||
from vllm.v1.attention.ops.common import cp_lse_ag_out_ar, cp_lse_ag_out_rs
|
||||
@@ -375,6 +376,7 @@ class MLAAttention(nn.Module, AttentionLayerBase):
|
||||
use_sparse: bool = False,
|
||||
indexer: object | None = None,
|
||||
topk_indices_buffer: torch.Tensor | None = None,
|
||||
non_causal_multi_token_decode: bool = False,
|
||||
**extra_impl_args,
|
||||
):
|
||||
super().__init__()
|
||||
@@ -391,6 +393,7 @@ class MLAAttention(nn.Module, AttentionLayerBase):
|
||||
self.head_size = kv_lora_rank + qk_rope_head_dim
|
||||
self.layer_name = prefix
|
||||
self.indexer = indexer
|
||||
self.non_causal_multi_token_decode = non_causal_multi_token_decode
|
||||
self.num_kv_heads = 1
|
||||
self.qk_head_dim = self.qk_nope_head_dim + self.qk_rope_head_dim
|
||||
|
||||
@@ -1123,6 +1126,7 @@ class MLAAttention(nn.Module, AttentionLayerBase):
|
||||
dtype=kv_cache_dtype,
|
||||
cache_dtype_str=self.kv_cache_dtype,
|
||||
kv_quant_mode=get_kv_quant_mode(self.kv_cache_dtype),
|
||||
non_causal_multi_token_decode=self.non_causal_multi_token_decode,
|
||||
)
|
||||
|
||||
def _v_up_proj(self, x: torch.Tensor, out: torch.Tensor):
|
||||
@@ -1450,6 +1454,8 @@ class MLACommonMetadata(AttentionMetadata, Generic[D]):
|
||||
num_decode_tokens: int
|
||||
num_prefills: int
|
||||
|
||||
causal: bool = True
|
||||
|
||||
# The dimension of the attention heads
|
||||
head_dim: int | None = None
|
||||
|
||||
@@ -1724,6 +1730,9 @@ class MLACommonMetadataBuilder(AttentionMetadataBuilder[M]):
|
||||
# speculative decoding is enabled.
|
||||
query_len_support: ClassVar[QueryLenSupport] = QueryLenSupport.SINGLE_ONLY
|
||||
|
||||
# Whether this builder can flatten a non-causal query block into decode rows.
|
||||
supports_non_causal_multi_token_decode: ClassVar[bool] = False
|
||||
|
||||
# The threshold for reordering the batch into decode and prefill requests.
|
||||
# If > 1, the batch will be reordered such that requests with
|
||||
# query length <= threshold are classified as decode requests.
|
||||
@@ -1824,8 +1833,14 @@ class MLACommonMetadataBuilder(AttentionMetadataBuilder[M]):
|
||||
self.vllm_config = vllm_config
|
||||
self.device = device
|
||||
self.use_pcp = parallel_config.prefill_context_parallel_size > 1
|
||||
self.non_causal_multi_token_decode = getattr(
|
||||
kv_cache_spec, "non_causal_multi_token_decode", False
|
||||
)
|
||||
|
||||
self.num_heads = self.model_config.get_num_attention_heads(parallel_config)
|
||||
# A draft cache group can have a different head count from the target.
|
||||
self.num_heads = get_num_attention_heads_from_layers(
|
||||
vllm_config, layer_names
|
||||
) or self.model_config.get_num_attention_heads(parallel_config)
|
||||
self.mla_dims = get_mla_dims(self.model_config)
|
||||
self.aot_schedule = current_platform.is_cuda()
|
||||
|
||||
@@ -1951,14 +1966,42 @@ class MLACommonMetadataBuilder(AttentionMetadataBuilder[M]):
|
||||
seq_lens = common_attn_metadata.seq_lens
|
||||
dcp_local_seq_lens = common_attn_metadata.dcp_local_seq_lens
|
||||
|
||||
num_decodes, num_prefills, num_decode_tokens, num_prefill_tokens = (
|
||||
split_decodes_and_prefills(
|
||||
common_attn_metadata,
|
||||
decode_threshold=self.reorder_batch_threshold,
|
||||
require_uniform=(self.query_len_support != QueryLenSupport.VARLEN),
|
||||
treat_short_extends_as_decodes=not self.use_pcp,
|
||||
non_causal_decode = common_attn_metadata.causal is False
|
||||
if non_causal_decode:
|
||||
if not (
|
||||
self.supports_non_causal_multi_token_decode
|
||||
and self.non_causal_multi_token_decode
|
||||
):
|
||||
raise ValueError(
|
||||
"Non-causal multi-token MLA requires an explicitly supported "
|
||||
"attention group."
|
||||
)
|
||||
query_lens = query_start_loc_cpu[1:] - query_start_loc_cpu[:-1]
|
||||
num_active_reqs = int(torch.count_nonzero(query_lens > 0))
|
||||
uniform_active_queries = num_active_reqs > 0 and bool(
|
||||
torch.all(query_lens[:num_active_reqs] == query_lens[0])
|
||||
)
|
||||
trailing_graph_padding = bool(torch.all(query_lens[num_active_reqs:] == 0))
|
||||
if not (uniform_active_queries and trailing_graph_padding):
|
||||
raise ValueError(
|
||||
"Non-causal MLA requires a uniform query block; got query "
|
||||
f"lengths {query_lens.tolist()}."
|
||||
)
|
||||
# Use exact GPU sequence lengths instead of the prefill path's CPU
|
||||
# context-length upper bounds.
|
||||
num_decodes = num_reqs
|
||||
num_prefills = 0
|
||||
num_decode_tokens = num_tokens
|
||||
num_prefill_tokens = 0
|
||||
else:
|
||||
num_decodes, num_prefills, num_decode_tokens, num_prefill_tokens = (
|
||||
split_decodes_and_prefills(
|
||||
common_attn_metadata,
|
||||
decode_threshold=self.reorder_batch_threshold,
|
||||
require_uniform=(self.query_len_support != QueryLenSupport.VARLEN),
|
||||
treat_short_extends_as_decodes=not self.use_pcp,
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
assert num_decodes + num_prefills == num_reqs
|
||||
assert num_decode_tokens + num_prefill_tokens == num_tokens
|
||||
@@ -2049,6 +2092,7 @@ class MLACommonMetadataBuilder(AttentionMetadataBuilder[M]):
|
||||
num_decodes=num_decodes,
|
||||
num_decode_tokens=num_decode_tokens,
|
||||
num_prefills=num_prefills,
|
||||
causal=not non_causal_decode,
|
||||
prefill=prefill_metadata,
|
||||
decode=decode_metadata,
|
||||
)
|
||||
|
||||
@@ -505,7 +505,7 @@ def deepgemm_moe_permute(
|
||||
dtype=torch.int32,
|
||||
)
|
||||
else:
|
||||
aq_scale_out = torch.empty((M_sum, sf_k), device=device, dtype=torch.float32)
|
||||
aq_scale_out = torch.zeros((M_sum, sf_k), device=device, dtype=torch.float32)
|
||||
|
||||
# DeepGEMM uses negative values in m_indices (here expert_ids) to mark
|
||||
# completely invalid / padded blocks that should be skipped. We always
|
||||
|
||||
@@ -204,6 +204,7 @@ def persistent_masked_m_silu_mul_quant(
|
||||
dtype=ys_dtype,
|
||||
device=y.device,
|
||||
)
|
||||
y_s.zero_()
|
||||
|
||||
ceil_ue8m0 = quant_scale_fmt in [
|
||||
DeepGemmQuantScaleFMT.FLOAT32_CEIL_UE8M0,
|
||||
|
||||
@@ -671,10 +671,10 @@ class MarlinExpertsBase(mk.FusedMoEExpertsModular):
|
||||
|
||||
@staticmethod
|
||||
def _supports_parallel_config(moe_parallel_config: FusedMoEParallelConfig) -> bool:
|
||||
return not (
|
||||
moe_parallel_config.use_fi_nvl_two_sided_kernels
|
||||
or moe_parallel_config.use_fi_nvl_one_sided_kernels
|
||||
)
|
||||
# One-sided FI-NVL all2all pairs with MarlinExperts fine (the
|
||||
# compressed-tensors MXFP4 path runs this exact combo); only the
|
||||
# two-sided kernels are unsupported here.
|
||||
return not moe_parallel_config.use_fi_nvl_two_sided_kernels
|
||||
|
||||
@property
|
||||
def quant_type_id(self) -> int:
|
||||
@@ -1068,11 +1068,8 @@ class BatchedMarlinExperts(MarlinExpertsBase):
|
||||
return
|
||||
|
||||
num_experts, max_num_tokens = hidden_states.shape[:2]
|
||||
beta = activation_situ_beta
|
||||
beta = 1.0 if activation_situ_beta is None else activation_situ_beta
|
||||
linear_beta = activation_situ_linear_beta
|
||||
assert beta is not None, (
|
||||
"SITU requires activation_situ_beta from FusedMoEConfig"
|
||||
)
|
||||
torch.ops._C.masked_situ_and_mul(
|
||||
act_output.view(num_experts, max_num_tokens, -1),
|
||||
act_input.view(num_experts, max_num_tokens, -1),
|
||||
|
||||
@@ -5,6 +5,7 @@ from functools import lru_cache
|
||||
|
||||
import torch
|
||||
|
||||
import vllm.envs as envs
|
||||
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
@@ -262,6 +263,8 @@ def rocm_aiter_fused_experts(
|
||||
elif activation == MoEActivation.SWIGLUOAI_UNINTERLEAVE:
|
||||
activation_method = rocm_aiter_ops.get_aiter_activation_type("swiglu")
|
||||
activation_interleave = False
|
||||
elif activation == MoEActivation.SITU:
|
||||
activation_method = rocm_aiter_ops.get_aiter_activation_type("situ")
|
||||
else:
|
||||
raise ValueError(f"Unsupported activation: {activation}")
|
||||
|
||||
@@ -356,10 +359,12 @@ def rocm_aiter_fused_experts(
|
||||
# https://github.com/ROCm/aiter/blob/v0.1.13.post1/aiter/fused_moe.py#L1099
|
||||
# TODO: Revisit this once we bump AITER to 0.1.15 with padding fixes
|
||||
# for CK/FlyDSL MoE GEMM e.g. https://github.com/ROCm/aiter/pull/3401
|
||||
hidden_pad = hidden_pad // 128 * 128
|
||||
intermediate_pad = (
|
||||
intermediate_pad // 64 * 64 * (2 if moe_config.tp_size == 1 else 1)
|
||||
)
|
||||
# SITU's A16W4 FlyDSL kernel pads per gate/up half; pass through unrounded.
|
||||
if activation != MoEActivation.SITU:
|
||||
hidden_pad = hidden_pad // 128 * 128
|
||||
intermediate_pad = (
|
||||
intermediate_pad // 64 * 64 * (2 if moe_config.tp_size == 1 else 1)
|
||||
)
|
||||
|
||||
# https://github.com/ROCm/aiter/pull/3123 specialized the AITER stage1 GEMMs
|
||||
# for interleaved vs separated gate and up weights.
|
||||
@@ -370,7 +375,15 @@ def rocm_aiter_fused_experts(
|
||||
from aiter.ops.flydsl.moe_common import GateMode
|
||||
|
||||
gate_mode = ""
|
||||
if quant_config.use_mxfp4_w4a16:
|
||||
if activation == MoEActivation.SITU:
|
||||
# a8w4 (AITER_SITUV2_A8W4=1) uses the gate/up-interleaved (_gui_)
|
||||
# fp8 flydsl kernels; default a16w4 SiTU stays separated.
|
||||
gate_mode = (
|
||||
GateMode.INTERLEAVE.value
|
||||
if envs.AITER_SITUV2_A8W4
|
||||
else GateMode.SEPARATED.value
|
||||
)
|
||||
elif quant_config.use_mxfp4_w4a16:
|
||||
gate_mode = GateMode.INTERLEAVE.value
|
||||
elif activation_interleave is not None:
|
||||
gate_mode = (
|
||||
@@ -401,6 +414,8 @@ def rocm_aiter_fused_experts(
|
||||
bias1=quant_config.w1_bias if quant_config.use_mxfp4_w4a16 else None,
|
||||
bias2=quant_config.w2_bias if quant_config.use_mxfp4_w4a16 else None,
|
||||
moe_sorting_dispatch_policy=moe_sorting_dispatch_policy,
|
||||
beta=moe_config.activation_situ_beta,
|
||||
linear_beta=moe_config.activation_situ_linear_beta,
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -15,6 +15,10 @@ from vllm.model_executor.layers.fused_moe.topk_weight_and_reduce import (
|
||||
TopKWeightAndReduceNoOP,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.utils import trtllm_moe_pack_topk_ids_weights
|
||||
from vllm.model_executor.layers.quantization.utils.flashinfer_utils import (
|
||||
activation_to_flashinfer_int,
|
||||
has_flashinfer_situ_activation,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
QuantKey,
|
||||
kMxfp4Static,
|
||||
@@ -133,19 +137,16 @@ class TrtLlmMxfp4ExpertsBase:
|
||||
|
||||
@staticmethod
|
||||
def _supports_activation(activation: MoEActivation) -> bool:
|
||||
if activation == MoEActivation.SITU:
|
||||
return has_flashinfer_situ_activation()
|
||||
return activation in (
|
||||
MoEActivation.SWIGLUOAI,
|
||||
MoEActivation.SILU,
|
||||
MoEActivation.SITU,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _flashinfer_activation_type(activation: MoEActivation) -> int:
|
||||
from flashinfer.fused_moe.core import ActivationType
|
||||
|
||||
if activation == MoEActivation.SITU:
|
||||
return ActivationType.Situ.value
|
||||
return ActivationType.Swiglu.value
|
||||
return activation_to_flashinfer_int(activation)
|
||||
|
||||
@staticmethod
|
||||
def activation_format() -> mk.FusedMoEActivationFormat:
|
||||
@@ -315,6 +316,75 @@ class TrtLlmMxfp4ExpertsModular(TrtLlmMxfp4ExpertsBase, mk.FusedMoEExpertsModula
|
||||
output = (M, self.hidden_dim_unpadded)
|
||||
return (workspace1, workspace2, output)
|
||||
|
||||
def _max_supported_tokens(self, top_k: int, global_num_experts: int) -> int:
|
||||
"""Max tokens per kernel call before the batched-GEMM grid overflows.
|
||||
|
||||
The TRTLLM-Gen batched GEMM launches a static grid whose batch (Y)
|
||||
dimension is ``getMaxNumCtasInBatchDim(num_tokens, top_k, num_experts,
|
||||
tileTokensDim)`` and must stay <= 65535. Solving that for num_tokens
|
||||
with the smallest tile the kernel may pick (tileTokensDim=8, the runner
|
||||
default) gives a bound that is safe regardless of the tactic selected.
|
||||
Without it, large batches (e.g. Kimi-K3 top_k=16, EP16 profiling with
|
||||
131072 gathered tokens) overflow the grid and the GEMM launch fails.
|
||||
"""
|
||||
MAX_GRID_Y = 65535
|
||||
MIN_TILE_TOKENS_DIM = 8
|
||||
max_tokens = (MAX_GRID_Y - global_num_experts) * MIN_TILE_TOKENS_DIM // top_k
|
||||
return max(1, min(300000, max_tokens))
|
||||
|
||||
def _invoke_kernel(
|
||||
self,
|
||||
output: torch.Tensor,
|
||||
x_quant: torch.Tensor,
|
||||
x_scale: torch.Tensor | None,
|
||||
topk_ids: torch.Tensor,
|
||||
topk_weights: torch.Tensor,
|
||||
w1: torch.Tensor,
|
||||
w2: torch.Tensor,
|
||||
activation: MoEActivation,
|
||||
global_num_experts: int,
|
||||
local_num_experts: int,
|
||||
local_expert_offset: int,
|
||||
topk: int,
|
||||
) -> None:
|
||||
from flashinfer import trtllm_fp4_block_scale_routed_moe
|
||||
|
||||
packed_tensor = trtllm_moe_pack_topk_ids_weights(topk_ids, topk_weights)
|
||||
trtllm_fp4_block_scale_routed_moe(
|
||||
topk_ids=packed_tensor,
|
||||
routing_bias=None,
|
||||
hidden_states=x_quant,
|
||||
hidden_states_scale=x_scale,
|
||||
gemm1_weights=w1,
|
||||
gemm1_weights_scale=self.w1_scale,
|
||||
gemm1_bias=self.w1_bias,
|
||||
gemm1_alpha=self.gemm1_alpha,
|
||||
gemm1_beta=self.gemm1_beta,
|
||||
gemm1_clamp_limit=self.gemm1_clamp_limit,
|
||||
gemm2_weights=w2,
|
||||
gemm2_weights_scale=self.w2_scale,
|
||||
gemm2_bias=self.w2_bias,
|
||||
output1_scale_scalar=None,
|
||||
output1_scale_gate_scalar=None,
|
||||
output2_scale_scalar=None,
|
||||
num_experts=global_num_experts,
|
||||
top_k=topk,
|
||||
n_group=None,
|
||||
topk_group=None,
|
||||
intermediate_size=self.intermediate_size_per_partition,
|
||||
local_expert_offset=local_expert_offset,
|
||||
local_num_experts=local_num_experts,
|
||||
routed_scaling_factor=None,
|
||||
# Modular kernel receives pre-routed tokens, so routing is already
|
||||
# done. Use Renormalize as a safe default the TRTLLM kernel supports.
|
||||
routing_method_type=RoutingMethodType.Renormalize,
|
||||
do_finalize=True,
|
||||
enable_pdl=True,
|
||||
activation_type=self._flashinfer_activation_type(activation),
|
||||
output=output,
|
||||
tune_max_num_tokens=max(self.max_capture_size, 1),
|
||||
)
|
||||
|
||||
def apply(
|
||||
self,
|
||||
output: torch.Tensor,
|
||||
@@ -335,7 +405,6 @@ class TrtLlmMxfp4ExpertsModular(TrtLlmMxfp4ExpertsBase, mk.FusedMoEExpertsModula
|
||||
):
|
||||
topk = topk_ids.size(-1)
|
||||
local_num_experts = w1.size(0)
|
||||
intermediate_size = self.intermediate_size_per_partition
|
||||
local_expert_offset = self.moe_config.ep_rank * local_num_experts
|
||||
|
||||
if a1q_scale is not None:
|
||||
@@ -346,49 +415,27 @@ class TrtLlmMxfp4ExpertsModular(TrtLlmMxfp4ExpertsBase, mk.FusedMoEExpertsModula
|
||||
x_quant = hidden_states
|
||||
x_scale = None
|
||||
|
||||
# Pack topk ids and weights into format expected by the kernel.
|
||||
packed_tensor = trtllm_moe_pack_topk_ids_weights(topk_ids, topk_weights)
|
||||
|
||||
assert self.w1_scale is not None
|
||||
assert self.w2_scale is not None
|
||||
kwargs = {
|
||||
"topk_ids": packed_tensor,
|
||||
"routing_bias": None,
|
||||
"hidden_states": x_quant,
|
||||
"hidden_states_scale": x_scale,
|
||||
"gemm1_weights": w1,
|
||||
"gemm1_weights_scale": self.w1_scale,
|
||||
"gemm1_bias": self.w1_bias,
|
||||
"gemm1_alpha": self.gemm1_alpha,
|
||||
"gemm1_beta": self.gemm1_beta,
|
||||
"gemm1_clamp_limit": self.gemm1_clamp_limit,
|
||||
"gemm2_weights": w2,
|
||||
"gemm2_weights_scale": self.w2_scale,
|
||||
"gemm2_bias": self.w2_bias,
|
||||
"output1_scale_scalar": None,
|
||||
"output1_scale_gate_scalar": None,
|
||||
"output2_scale_scalar": None,
|
||||
"num_experts": global_num_experts,
|
||||
"top_k": topk,
|
||||
"n_group": None,
|
||||
"topk_group": None,
|
||||
"intermediate_size": intermediate_size,
|
||||
"local_expert_offset": local_expert_offset,
|
||||
"local_num_experts": local_num_experts,
|
||||
"routed_scaling_factor": None,
|
||||
# Modular kernel receives pre-routed tokens, so routing
|
||||
# is already done. Use Renormalize as a safe default that
|
||||
# the TRTLLM C++ kernel supports.
|
||||
"routing_method_type": RoutingMethodType.Renormalize,
|
||||
"do_finalize": True,
|
||||
"enable_pdl": True,
|
||||
"activation_type": self._flashinfer_activation_type(activation),
|
||||
"output": output,
|
||||
"tune_max_num_tokens": max(self.max_capture_size, 1),
|
||||
}
|
||||
|
||||
from flashinfer import trtllm_fp4_block_scale_routed_moe
|
||||
|
||||
trtllm_fp4_block_scale_routed_moe(**kwargs)
|
||||
# Chunk tokens so the batched-GEMM grid stays within CUDA limits.
|
||||
M = x_quant.size(0)
|
||||
chunk_size = self._max_supported_tokens(topk, global_num_experts)
|
||||
for start in range(0, M, chunk_size):
|
||||
end = min(start + chunk_size, M)
|
||||
self._invoke_kernel(
|
||||
output[start:end],
|
||||
x_quant[start:end],
|
||||
None if x_scale is None else x_scale[start:end],
|
||||
topk_ids[start:end],
|
||||
topk_weights[start:end],
|
||||
w1,
|
||||
w2,
|
||||
activation,
|
||||
global_num_experts,
|
||||
local_num_experts,
|
||||
local_expert_offset,
|
||||
topk,
|
||||
)
|
||||
|
||||
return output
|
||||
|
||||
@@ -22,6 +22,7 @@ from vllm.model_executor.layers.fused_moe.utils import (
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.flashinfer_utils import (
|
||||
activation_to_flashinfer_int,
|
||||
has_flashinfer_situ_activation,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
QuantKey,
|
||||
@@ -225,15 +226,15 @@ class TrtLlmNvFp4ExpertsBase:
|
||||
|
||||
@staticmethod
|
||||
def _supports_activation(activation: MoEActivation) -> bool:
|
||||
"""Supports SiLU, RELU^2 non-gated, GELU, clamped SwiGLU-OAI, and the
|
||||
private SituGLU (SITU) cubin."""
|
||||
"""Supports SITU only when the installed FlashInfer exposes it."""
|
||||
if activation == MoEActivation.SITU:
|
||||
return has_flashinfer_situ_activation()
|
||||
return activation in [
|
||||
MoEActivation.SILU,
|
||||
MoEActivation.RELU2_NO_MUL,
|
||||
MoEActivation.GELU,
|
||||
MoEActivation.GELU_TANH,
|
||||
MoEActivation.SWIGLUOAI_UNINTERLEAVE,
|
||||
MoEActivation.SITU,
|
||||
]
|
||||
|
||||
@staticmethod
|
||||
|
||||
@@ -648,7 +648,7 @@ def mxfp4_round_up_hidden_size_and_intermediate_size(
|
||||
else:
|
||||
hidden_size = round_up(hidden_size, 256)
|
||||
elif backend in TRTLLM_BACKENDS:
|
||||
intermediate_size = round_up(intermediate_size, 256)
|
||||
intermediate_size = round_up(intermediate_size, 128)
|
||||
hidden_size = round_up(hidden_size, 256)
|
||||
elif backend in (
|
||||
Mxfp4MoeBackend.FLASHINFER_CUTLASS_MXFP4_BF16,
|
||||
|
||||
@@ -102,7 +102,7 @@ class BatchedPrepareAndFinalize(mk.FusedMoEPrepareAndFinalizeModular):
|
||||
num_local_experts, self.max_num_tokens, hidden_dim
|
||||
)
|
||||
|
||||
b_a1_scale = torch.empty(scale_shape, dtype=torch.float32, device=a1.device)
|
||||
b_a1_scale = torch.zeros(scale_shape, dtype=torch.float32, device=a1.device)
|
||||
else:
|
||||
assert quant_config.a1_scale is None
|
||||
b_a1_scale = None
|
||||
|
||||
@@ -6,11 +6,6 @@ import vllm.envs as envs
|
||||
from vllm.config import get_current_vllm_config
|
||||
from vllm.distributed import tensor_model_parallel_all_reduce
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.fused_allreduce_gemma_rms_norm import (
|
||||
_AR_RESIDUAL_RMS_NORM,
|
||||
_can_use_flashinfer,
|
||||
flashinfer_trtllm_fused_allreduce_norm,
|
||||
)
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.utils.torch_utils import aux_stream, current_stream
|
||||
|
||||
@@ -82,25 +77,30 @@ class LatentMoERunner(MoERunner):
|
||||
)
|
||||
self._k3_latent_moe_tail_op = op
|
||||
|
||||
def _get_zero_residual(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
||||
def _get_zero_residual(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
max_token_num: int,
|
||||
) -> torch.Tensor:
|
||||
"""Read-only zero ``residual_in`` for the fused AR+RMSNorm kernel.
|
||||
|
||||
flashinfer requires a residual buffer even when there is no residual to
|
||||
add.
|
||||
"""
|
||||
numel = hidden_states.numel()
|
||||
buf = getattr(self, "_zero_residual", None)
|
||||
if (
|
||||
buf is None
|
||||
or buf.dtype != hidden_states.dtype
|
||||
or buf.device != hidden_states.device
|
||||
or buf.numel() < numel
|
||||
):
|
||||
if buf is None:
|
||||
buf = torch.zeros(
|
||||
numel, dtype=hidden_states.dtype, device=hidden_states.device
|
||||
max_token_num * hidden_states.shape[-1],
|
||||
dtype=hidden_states.dtype,
|
||||
device=hidden_states.device,
|
||||
)
|
||||
self._zero_residual = buf
|
||||
return buf[:numel].view_as(hidden_states)
|
||||
|
||||
assert buf.dtype == hidden_states.dtype
|
||||
assert buf.device == hidden_states.device
|
||||
assert hidden_states.numel() <= buf.numel()
|
||||
|
||||
return buf[: hidden_states.numel()].view_as(hidden_states)
|
||||
|
||||
def _use_fused_path(self) -> bool:
|
||||
# The fused path merges the latent and shared reductions into one
|
||||
@@ -224,6 +224,12 @@ class LatentMoERunner(MoERunner):
|
||||
norm: RMSNorm,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""All-reduce + add residual + (standard) RMSNorm, fused via flashinfer."""
|
||||
from vllm.model_executor.layers.fused_allreduce_gemma_rms_norm import (
|
||||
_AR_RESIDUAL_RMS_NORM,
|
||||
_can_use_flashinfer,
|
||||
flashinfer_trtllm_fused_allreduce_norm,
|
||||
)
|
||||
|
||||
if self.moe_config.tp_size == 1:
|
||||
return norm(hidden_states)
|
||||
|
||||
@@ -238,7 +244,7 @@ class LatentMoERunner(MoERunner):
|
||||
# buffer and the normalized result into norm_out.
|
||||
flashinfer_trtllm_fused_allreduce_norm(
|
||||
allreduce_in=hidden_states,
|
||||
residual=self._get_zero_residual(hidden_states),
|
||||
residual=self._get_zero_residual(hidden_states, max_token_num),
|
||||
rms_gamma=norm.weight,
|
||||
rms_eps=norm.variance_epsilon,
|
||||
world_size=self.moe_config.tp_size,
|
||||
|
||||
@@ -1,32 +1,33 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from collections.abc import Callable
|
||||
|
||||
import torch
|
||||
from einops import rearrange
|
||||
from torch import nn
|
||||
from torch.nn.parameter import Parameter
|
||||
|
||||
from vllm.config import VllmConfig, get_current_vllm_config
|
||||
from vllm.distributed import (
|
||||
divide,
|
||||
)
|
||||
from vllm.forward_context import ForwardContext, get_forward_context
|
||||
from vllm.logger import init_logger
|
||||
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed import divide, get_tensor_model_parallel_rank
|
||||
from vllm.forward_context import get_forward_context
|
||||
from vllm.model_executor.custom_op import PluggableLayer
|
||||
from vllm.model_executor.layers.mamba.gdn.base import GatedDeltaNetAttention
|
||||
from vllm.model_executor.model_loader.weight_utils import sharded_weight_loader
|
||||
from vllm.model_executor.utils import set_weight_attrs
|
||||
from vllm.third_party.flash_linear_attention.ops.kda import (
|
||||
FusedRMSNormGated,
|
||||
chunk_kda_with_fused_gate,
|
||||
fused_kda_gate,
|
||||
fused_recurrent_kda,
|
||||
from vllm.model_executor.model_loader.weight_utils import (
|
||||
default_weight_loader,
|
||||
sharded_weight_loader,
|
||||
)
|
||||
from vllm.model_executor.parameter import BasevLLMParameter
|
||||
from vllm.model_executor.utils import set_weight_attrs
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.third_party.flash_linear_attention.ops.kda import FusedRMSNormGated
|
||||
from vllm.transformers_utils.configs.kimi_linear import KimiLinearConfig
|
||||
from vllm.utils.torch_utils import direct_register_custom_op
|
||||
from vllm.v1.attention.backends.gdn_attn import GDNAttentionMetadata
|
||||
|
||||
from ...linear import (
|
||||
ColumnParallelLinear,
|
||||
MergedColumnParallelLinear,
|
||||
ReplicatedLinear,
|
||||
RowParallelLinear,
|
||||
)
|
||||
@@ -36,49 +37,118 @@ from ..mamba_utils import (
|
||||
is_conv_state_dim_first,
|
||||
)
|
||||
from ..ops.causal_conv1d import causal_conv1d_fn, causal_conv1d_update
|
||||
from ..ops.gather_initial_states import gather_initial_states
|
||||
|
||||
logger = init_logger(__name__)
|
||||
# Empirical lower bound for the KDA gate to avoid numerical underflow.
|
||||
_KDA_GATE_LOGBOUND_MIN = -5.0
|
||||
|
||||
|
||||
def kda_attention(
|
||||
q_proj_states: torch.Tensor,
|
||||
k_proj_states: torch.Tensor,
|
||||
v_proj_states: torch.Tensor,
|
||||
g1: torch.Tensor,
|
||||
beta: torch.Tensor,
|
||||
core_attn_out: torch.Tensor,
|
||||
layer_name: str,
|
||||
) -> None:
|
||||
forward_context: ForwardContext = get_forward_context()
|
||||
self = forward_context.no_compile_layers[layer_name]
|
||||
self._forward(
|
||||
q_proj_states=q_proj_states,
|
||||
k_proj_states=k_proj_states,
|
||||
v_proj_states=v_proj_states,
|
||||
g1=g1,
|
||||
beta=beta,
|
||||
core_attn_out=core_attn_out,
|
||||
)
|
||||
def a_log_weight_loader(
|
||||
shard_axis: int,
|
||||
) -> Callable[[torch.Tensor, torch.Tensor], None]:
|
||||
"""Load KDA A_log stored as either old 4D or current 1D weights."""
|
||||
|
||||
def loader(param: torch.Tensor, loaded_weight: torch.Tensor) -> None:
|
||||
tp_rank = get_tensor_model_parallel_rank()
|
||||
shard_size = param.data.shape[shard_axis]
|
||||
start_idx = tp_rank * shard_size
|
||||
|
||||
if loaded_weight.dim() == 4:
|
||||
assert loaded_weight.shape[:2] == (1, 1), (
|
||||
f"Expected old A_log shape (1, 1, H, 1), got {loaded_weight.shape}"
|
||||
)
|
||||
assert loaded_weight.shape[-1] == 1, (
|
||||
f"Expected old A_log last dim to be 1, got {loaded_weight.shape}"
|
||||
)
|
||||
loaded_weight = loaded_weight.view(loaded_weight.shape[2])
|
||||
|
||||
loaded_weight = loaded_weight.narrow(shard_axis, start_idx, shard_size)
|
||||
return default_weight_loader(param, loaded_weight)
|
||||
|
||||
return loader
|
||||
|
||||
|
||||
def kda_attention_fake(
|
||||
q_proj_states: torch.Tensor,
|
||||
k_proj_states: torch.Tensor,
|
||||
v_proj_states: torch.Tensor,
|
||||
g1: torch.Tensor,
|
||||
beta: torch.Tensor,
|
||||
core_attn_out: torch.Tensor,
|
||||
layer_name: str,
|
||||
) -> None:
|
||||
return
|
||||
def _make_fused_conv1d_weight_loader(
|
||||
dims: list[int],
|
||||
tp_size: int,
|
||||
tp_rank: int,
|
||||
) -> Callable[..., None]:
|
||||
sharded_dims = [dim // tp_size for dim in dims]
|
||||
|
||||
def weight_loader(
|
||||
param: torch.Tensor,
|
||||
loaded_weight: torch.Tensor,
|
||||
loaded_shard_id: int,
|
||||
) -> None:
|
||||
if loaded_weight.dim() == 2:
|
||||
loaded_weight = loaded_weight.unsqueeze(1)
|
||||
shard_size = sharded_dims[loaded_shard_id]
|
||||
source_start = tp_rank * shard_size
|
||||
target_start = sum(sharded_dims[:loaded_shard_id])
|
||||
loaded_shard = loaded_weight[source_start : source_start + shard_size]
|
||||
param.data[target_start : target_start + shard_size].copy_(loaded_shard)
|
||||
|
||||
return weight_loader
|
||||
|
||||
|
||||
direct_register_custom_op(
|
||||
op_name="kda_attention",
|
||||
op_func=kda_attention,
|
||||
mutates_args=["core_attn_out"],
|
||||
fake_impl=kda_attention_fake,
|
||||
)
|
||||
class _KimiGDNMergedColumnParallelLinear(MergedColumnParallelLinear):
|
||||
"""Merged projection with one output replicated across TP ranks.
|
||||
|
||||
The replicated shard is represented as ``size * tp_size`` so the merged
|
||||
parameter reserves ``size`` local rows on every rank. Loading that shard
|
||||
from rank zero then gives every rank the complete checkpoint weight.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
input_size: int,
|
||||
output_sizes: list[int],
|
||||
replicated_shard_id: int,
|
||||
tp_size: int,
|
||||
**kwargs,
|
||||
) -> None:
|
||||
self.replicated_shard_id = replicated_shard_id
|
||||
output_sizes = output_sizes.copy()
|
||||
output_sizes[replicated_shard_id] *= tp_size
|
||||
super().__init__(input_size, output_sizes, **kwargs)
|
||||
|
||||
def weight_loader(
|
||||
self,
|
||||
param: Parameter,
|
||||
loaded_weight: torch.Tensor,
|
||||
loaded_shard_id: tuple[int, ...] | int | None = None,
|
||||
) -> None:
|
||||
tp_rank = self.tp_rank
|
||||
param_tp_rank = getattr(param, "tp_rank", None)
|
||||
if loaded_shard_id == self.replicated_shard_id:
|
||||
self.tp_rank = 0
|
||||
if param_tp_rank is not None:
|
||||
param.tp_rank = 0
|
||||
try:
|
||||
super().weight_loader(param, loaded_weight, loaded_shard_id)
|
||||
finally:
|
||||
self.tp_rank = tp_rank
|
||||
if param_tp_rank is not None:
|
||||
param.tp_rank = param_tp_rank
|
||||
|
||||
def weight_loader_v2(
|
||||
self,
|
||||
param: BasevLLMParameter,
|
||||
loaded_weight: torch.Tensor,
|
||||
loaded_shard_id: tuple[int, ...] | int | None = None,
|
||||
) -> None:
|
||||
tp_rank = self.tp_rank
|
||||
param_tp_rank = getattr(param, "tp_rank", None)
|
||||
if loaded_shard_id == self.replicated_shard_id:
|
||||
self.tp_rank = 0
|
||||
if param_tp_rank is not None:
|
||||
param.tp_rank = 0
|
||||
try:
|
||||
super().weight_loader_v2(param, loaded_weight, loaded_shard_id)
|
||||
finally:
|
||||
self.tp_rank = tp_rank
|
||||
if param_tp_rank is not None:
|
||||
param.tp_rank = param_tp_rank
|
||||
|
||||
|
||||
@PluggableLayer.register("kimi_gated_delta_net_attention")
|
||||
@@ -96,7 +166,11 @@ class KimiGatedDeltaNetAttention(GatedDeltaNetAttention):
|
||||
self,
|
||||
) -> tuple[tuple[int, ...], tuple[int, ...]]:
|
||||
return MambaStateShapeCalculator.kda_state_shape(
|
||||
self.tp_size, self.num_heads, self.head_dim, conv_kernel_size=self.conv_size
|
||||
self.tp_size,
|
||||
self.num_heads,
|
||||
self.head_dim,
|
||||
conv_kernel_size=self.conv_size,
|
||||
num_spec=self.num_spec,
|
||||
)
|
||||
|
||||
def __init__(
|
||||
@@ -114,122 +188,142 @@ class KimiGatedDeltaNetAttention(GatedDeltaNetAttention):
|
||||
assert self.num_heads % self.tp_size == 0
|
||||
self.local_num_heads = divide(self.num_heads, self.tp_size)
|
||||
|
||||
projection_size = self.head_dim * self.num_heads
|
||||
self.projection_size = self.head_dim * self.num_heads
|
||||
self.local_projection_size = divide(self.projection_size, self.tp_size)
|
||||
self.conv_size = kda_config["short_conv_kernel_size"]
|
||||
self.use_full_rank_gate = kda_config.get("use_full_rank_gate", False)
|
||||
|
||||
self.q_proj = ColumnParallelLinear(
|
||||
if self.use_full_rank_gate:
|
||||
# Keep f_a before the narrow beta shard, then pad each TP-local row
|
||||
# to select the aligned BF16 GEMM path. The padding also avoids an
|
||||
# Inductor correctness issue seen with the row-strided G view.
|
||||
qkvg_output_sizes = [self.projection_size] * 4
|
||||
in_proj_output_sizes = qkvg_output_sizes + [
|
||||
self.head_dim,
|
||||
self.num_heads,
|
||||
]
|
||||
local_output_size = (
|
||||
4 * self.local_projection_size + self.head_dim + self.local_num_heads
|
||||
)
|
||||
self.in_proj_padding = -local_output_size % 16
|
||||
if self.in_proj_padding:
|
||||
in_proj_output_sizes.append(self.in_proj_padding * self.tp_size)
|
||||
else:
|
||||
in_proj_output_sizes = [self.projection_size] * 3 + [
|
||||
self.num_heads,
|
||||
self.head_dim,
|
||||
]
|
||||
self.in_proj_padding = 0
|
||||
self.in_proj_qkvgfab = _KimiGDNMergedColumnParallelLinear(
|
||||
self.hidden_size,
|
||||
projection_size,
|
||||
in_proj_output_sizes,
|
||||
replicated_shard_id=4,
|
||||
tp_size=self.tp_size,
|
||||
bias=False,
|
||||
quant_config=self.quant_config,
|
||||
prefix=f"{prefix}.q_proj",
|
||||
)
|
||||
self.k_proj = ColumnParallelLinear(
|
||||
self.hidden_size,
|
||||
projection_size,
|
||||
bias=False,
|
||||
quant_config=self.quant_config,
|
||||
prefix=f"{prefix}.k_proj",
|
||||
)
|
||||
self.v_proj = ColumnParallelLinear(
|
||||
self.hidden_size,
|
||||
projection_size,
|
||||
bias=False,
|
||||
quant_config=self.quant_config,
|
||||
prefix=f"{prefix}.v_proj",
|
||||
)
|
||||
|
||||
self.f_a_proj = ReplicatedLinear(
|
||||
self.hidden_size,
|
||||
self.head_dim,
|
||||
bias=False,
|
||||
quant_config=self.quant_config,
|
||||
prefix=f"{prefix}.f_a_proj",
|
||||
prefix=f"{prefix}.in_proj_qkvgfab",
|
||||
)
|
||||
if self.in_proj_padding:
|
||||
self.in_proj_qkvgfab.weight.data[-self.in_proj_padding :].zero_()
|
||||
|
||||
self.f_b_proj = ColumnParallelLinear(
|
||||
self.head_dim,
|
||||
projection_size,
|
||||
self.projection_size,
|
||||
bias=False,
|
||||
quant_config=self.quant_config,
|
||||
prefix=f"{prefix}.f_b_proj",
|
||||
)
|
||||
self.dt_bias = nn.Parameter(
|
||||
torch.empty(divide(projection_size, self.tp_size), dtype=torch.float32)
|
||||
torch.empty(self.local_projection_size, dtype=torch.float32)
|
||||
)
|
||||
|
||||
set_weight_attrs(self.dt_bias, {"weight_loader": sharded_weight_loader(0)})
|
||||
|
||||
self.b_proj = ColumnParallelLinear(
|
||||
self.hidden_size,
|
||||
self.num_heads,
|
||||
bias=False,
|
||||
quant_config=self.quant_config,
|
||||
prefix=f"{prefix}.b_proj",
|
||||
)
|
||||
|
||||
self.q_conv1d = ColumnParallelLinear(
|
||||
# One packed parameter and cache let decode run a single conv update.
|
||||
# Prefill slices them back into Q/K/V to obtain dense outputs cheaply.
|
||||
self.conv1d = ColumnParallelLinear(
|
||||
input_size=self.conv_size,
|
||||
output_size=projection_size,
|
||||
output_size=3 * self.projection_size,
|
||||
bias=False,
|
||||
params_dtype=torch.float32,
|
||||
prefix=f"{prefix}.q_conv1d",
|
||||
prefix=f"{prefix}.conv1d",
|
||||
)
|
||||
self.k_conv1d = ColumnParallelLinear(
|
||||
input_size=self.conv_size,
|
||||
output_size=projection_size,
|
||||
bias=False,
|
||||
params_dtype=torch.float32,
|
||||
prefix=f"{prefix}.k_conv1d",
|
||||
self.conv1d.weight.data = self.conv1d.weight.data.unsqueeze(1)
|
||||
delattr(self.conv1d.weight, "weight_loader")
|
||||
set_weight_attrs(
|
||||
self.conv1d.weight,
|
||||
{
|
||||
"weight_loader": _make_fused_conv1d_weight_loader(
|
||||
[self.projection_size] * 3,
|
||||
self.tp_size,
|
||||
self.tp_rank,
|
||||
)
|
||||
},
|
||||
)
|
||||
self.v_conv1d = ColumnParallelLinear(
|
||||
input_size=self.conv_size,
|
||||
output_size=projection_size,
|
||||
bias=False,
|
||||
params_dtype=torch.float32,
|
||||
prefix=f"{prefix}.v_conv1d",
|
||||
)
|
||||
# unsqueeze to fit conv1d weights shape into the linear weights shape.
|
||||
# Can't do this in `weight_loader` since it already exists in
|
||||
# `ColumnParallelLinear` and `set_weight_attrs`
|
||||
# doesn't allow to override it
|
||||
self.q_conv1d.weight.data = self.q_conv1d.weight.data.unsqueeze(1)
|
||||
self.k_conv1d.weight.data = self.k_conv1d.weight.data.unsqueeze(1)
|
||||
self.v_conv1d.weight.data = self.v_conv1d.weight.data.unsqueeze(1)
|
||||
|
||||
self.A_log = nn.Parameter(
|
||||
torch.empty(1, 1, self.local_num_heads, 1, dtype=torch.float32)
|
||||
torch.empty(self.local_num_heads, dtype=torch.float32)
|
||||
)
|
||||
set_weight_attrs(self.A_log, {"weight_loader": sharded_weight_loader(2)})
|
||||
set_weight_attrs(self.A_log, {"weight_loader": a_log_weight_loader(0)})
|
||||
|
||||
self.g_a_proj = ReplicatedLinear(
|
||||
self.hidden_size,
|
||||
self.head_dim,
|
||||
bias=False,
|
||||
quant_config=self.quant_config,
|
||||
prefix=f"{prefix}.g_a_proj",
|
||||
self.gate_lower_bound: float | None = kda_config.get("gate_lower_bound", None)
|
||||
if self.gate_lower_bound is not None:
|
||||
assert _KDA_GATE_LOGBOUND_MIN <= self.gate_lower_bound < 0, (
|
||||
"KDA gate lower bound must be in "
|
||||
f"[{_KDA_GATE_LOGBOUND_MIN}, 0). "
|
||||
f"Got {self.gate_lower_bound}."
|
||||
)
|
||||
self.use_safe_gate = self.gate_lower_bound is not None
|
||||
additional_config = vllm_config.additional_config
|
||||
backend = (
|
||||
additional_config.get("kda_prefill_backend", "auto")
|
||||
if isinstance(additional_config, dict)
|
||||
else "auto"
|
||||
)
|
||||
self.g_b_proj = ColumnParallelLinear(
|
||||
self.head_dim,
|
||||
projection_size,
|
||||
bias=False,
|
||||
quant_config=self.quant_config,
|
||||
prefix=f"{prefix}.g_b_proj",
|
||||
backend = "triton" if backend == "auto" else backend
|
||||
assert backend == "triton", (
|
||||
"The shared Kimi GDN layer only supports the Triton KDA "
|
||||
f"prefill backend, got {backend!r}."
|
||||
)
|
||||
if not self.use_full_rank_gate:
|
||||
self.g_a_proj = ReplicatedLinear(
|
||||
self.hidden_size,
|
||||
self.head_dim,
|
||||
bias=False,
|
||||
quant_config=self.quant_config,
|
||||
prefix=f"{prefix}.g_a_proj",
|
||||
)
|
||||
self.g_b_proj = ColumnParallelLinear(
|
||||
self.head_dim,
|
||||
self.projection_size,
|
||||
bias=False,
|
||||
quant_config=self.quant_config,
|
||||
prefix=f"{prefix}.g_b_proj",
|
||||
)
|
||||
self.o_norm = FusedRMSNormGated(self.head_dim, activation="sigmoid")
|
||||
self.o_proj = RowParallelLinear(
|
||||
projection_size,
|
||||
self.projection_size,
|
||||
self.hidden_size,
|
||||
bias=False,
|
||||
quant_config=self.quant_config,
|
||||
prefix=f"{prefix}.o_proj",
|
||||
)
|
||||
|
||||
compilation_config = get_current_vllm_config().compilation_config
|
||||
compilation_config = vllm_config.compilation_config
|
||||
if prefix in compilation_config.static_forward_context:
|
||||
raise ValueError(f"Duplicate layer name: {prefix}")
|
||||
compilation_config.static_forward_context[prefix] = self
|
||||
|
||||
def rearrange_mixed_qkv(
|
||||
self, mixed_qkv: torch.Tensor
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
seq_len = mixed_qkv.shape[0]
|
||||
qkv = mixed_qkv.view(seq_len, 3, self.local_num_heads, self.head_dim)
|
||||
# Materialize all three row-strided inputs with one token-major to
|
||||
# QKV-major permutation. Each unbound tensor is then contiguous.
|
||||
qkv = qkv.permute(1, 0, 2, 3).contiguous().unsqueeze(1)
|
||||
return qkv.unbind(0)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
@@ -237,42 +331,57 @@ class KimiGatedDeltaNetAttention(GatedDeltaNetAttention):
|
||||
output: torch.Tensor,
|
||||
) -> None:
|
||||
num_tokens = hidden_states.size(0)
|
||||
q = self.q_proj(hidden_states)[0]
|
||||
k = self.k_proj(hidden_states)[0]
|
||||
v = self.v_proj(hidden_states)[0]
|
||||
projected_qkvgfab = self.in_proj_qkvgfab(hidden_states)[0]
|
||||
if self.use_full_rank_gate:
|
||||
split_sizes = [
|
||||
3 * self.local_projection_size,
|
||||
self.local_projection_size,
|
||||
self.head_dim,
|
||||
self.local_num_heads,
|
||||
]
|
||||
if self.in_proj_padding:
|
||||
split_sizes.append(self.in_proj_padding)
|
||||
projected = projected_qkvgfab.split(split_sizes, dim=-1)
|
||||
mixed_qkv, g_proj_states, f_a, beta = projected[:4]
|
||||
else:
|
||||
mixed_qkv, beta, f_a = projected_qkvgfab.split(
|
||||
[
|
||||
3 * self.local_projection_size,
|
||||
self.local_num_heads,
|
||||
self.head_dim,
|
||||
],
|
||||
dim=-1,
|
||||
)
|
||||
g_proj_states = self.g_b_proj(self.g_a_proj(hidden_states)[0])[0]
|
||||
|
||||
beta = self.b_proj(hidden_states)[0].float().sigmoid()
|
||||
g1 = self.f_b_proj(self.f_a_proj(hidden_states)[0])[0]
|
||||
g1 = self.f_b_proj(f_a)[0]
|
||||
beta = beta.unsqueeze(0)
|
||||
g1 = rearrange(g1, "n (h d) -> 1 n h d", d=self.head_dim)
|
||||
|
||||
g_proj_states = self.g_b_proj(self.g_a_proj(hidden_states)[0])[0]
|
||||
g2 = rearrange(g_proj_states, "... (h d) -> ... h d", d=self.head_dim)
|
||||
|
||||
core_attn_out = torch.zeros(
|
||||
core_attn_out = torch.empty(
|
||||
(1, num_tokens, self.local_num_heads, self.head_dim),
|
||||
dtype=hidden_states.dtype,
|
||||
device=hidden_states.device,
|
||||
)
|
||||
torch.ops.vllm.kda_attention(
|
||||
q,
|
||||
k,
|
||||
v,
|
||||
g1,
|
||||
beta,
|
||||
core_attn_out,
|
||||
self.prefix,
|
||||
|
||||
self._forward(
|
||||
mixed_qkv=mixed_qkv,
|
||||
g1=g1,
|
||||
g2=g2,
|
||||
beta=beta,
|
||||
core_attn_out=core_attn_out,
|
||||
)
|
||||
core_attn_out = self.o_norm(core_attn_out, g2)
|
||||
core_attn_out = rearrange(core_attn_out, "1 n h d -> n (h d)")
|
||||
output[:] = self.o_proj(core_attn_out)[0]
|
||||
|
||||
@eager_break_during_capture
|
||||
def _forward(
|
||||
self,
|
||||
q_proj_states: torch.Tensor,
|
||||
k_proj_states: torch.Tensor,
|
||||
v_proj_states: torch.Tensor,
|
||||
mixed_qkv: torch.Tensor,
|
||||
g1: torch.Tensor,
|
||||
g2: torch.Tensor,
|
||||
beta: torch.Tensor,
|
||||
core_attn_out: torch.Tensor,
|
||||
) -> None:
|
||||
@@ -280,165 +389,247 @@ class KimiGatedDeltaNetAttention(GatedDeltaNetAttention):
|
||||
attn_metadata_raw = forward_context.attn_metadata
|
||||
|
||||
if attn_metadata_raw is None:
|
||||
# # V1 profile run
|
||||
return
|
||||
|
||||
# Vendor-specific KDA kernels: AMD/ROCm and NVIDIA keep their own copies
|
||||
# under kimi_k3/{amd,nvidia}/ops so each can diverge independently.
|
||||
if current_platform.is_rocm():
|
||||
from vllm.models.kimi_k3.amd.ops.third_party.kda import (
|
||||
chunk_kda_with_fused_gate,
|
||||
fused_recurrent_kda,
|
||||
fused_recurrent_kda_packed_decode,
|
||||
)
|
||||
else:
|
||||
from vllm.models.kimi_k3.nvidia.ops.third_party.kda import (
|
||||
chunk_kda_with_fused_gate,
|
||||
fused_recurrent_kda,
|
||||
fused_recurrent_kda_packed_decode,
|
||||
)
|
||||
|
||||
assert isinstance(attn_metadata_raw, dict)
|
||||
attn_metadata_narrowed = attn_metadata_raw[self.prefix]
|
||||
assert isinstance(attn_metadata_narrowed, GDNAttentionMetadata)
|
||||
has_initial_state = attn_metadata_narrowed.has_initial_state
|
||||
non_spec_query_start_loc = attn_metadata_narrowed.non_spec_query_start_loc
|
||||
non_spec_state_indices_tensor = (
|
||||
attn_metadata_narrowed.non_spec_state_indices_tensor
|
||||
) # noqa: E501
|
||||
num_actual_tokens = attn_metadata_narrowed.num_actual_tokens
|
||||
constant_caches = self.kv_cache
|
||||
|
||||
q_proj_states = q_proj_states[:num_actual_tokens]
|
||||
k_proj_states = k_proj_states[:num_actual_tokens]
|
||||
v_proj_states = v_proj_states[:num_actual_tokens]
|
||||
m = attn_metadata_narrowed
|
||||
has_initial_state = m.has_initial_state
|
||||
non_spec_query_start_loc = m.non_spec_query_start_loc
|
||||
non_spec_state_indices_tensor = m.non_spec_state_indices_tensor
|
||||
spec_sequence_masks = m.spec_sequence_masks
|
||||
spec_token_indx = m.spec_token_indx
|
||||
non_spec_token_indx = m.non_spec_token_indx
|
||||
spec_state_indices_tensor = m.spec_state_indices_tensor
|
||||
spec_query_start_loc = m.spec_query_start_loc
|
||||
num_accepted_tokens = m.num_accepted_tokens
|
||||
num_actual_tokens = m.num_actual_tokens
|
||||
mixed_qkv = mixed_qkv[:num_actual_tokens]
|
||||
g1 = g1[:, :num_actual_tokens]
|
||||
beta = beta[:, :num_actual_tokens]
|
||||
|
||||
(conv_state, recurrent_state) = constant_caches
|
||||
constant_caches = self.kv_cache
|
||||
|
||||
conv_state, recurrent_state = constant_caches
|
||||
# conv_state must be (..., dim, width-1) for the conv kernels.
|
||||
# DS layout stores it that way directly; SD layout needs a transpose.
|
||||
if not is_conv_state_dim_first():
|
||||
conv_state = conv_state.transpose(-1, -2)
|
||||
|
||||
conv_state_q, conv_state_k, conv_state_v = conv_state.chunk(3, dim=-2)
|
||||
conv_weights = self.conv1d.weight.view(
|
||||
self.conv1d.weight.size(0), self.conv1d.weight.size(2)
|
||||
)
|
||||
q_conv_weight, k_conv_weight, v_conv_weight = conv_weights.split(
|
||||
self.local_projection_size, dim=0
|
||||
)
|
||||
q_conv_state, k_conv_state, v_conv_state = conv_state.split(
|
||||
self.local_projection_size, dim=-2
|
||||
)
|
||||
|
||||
q_conv_weights = self.q_conv1d.weight.view(
|
||||
self.q_conv1d.weight.size(0), self.q_conv1d.weight.size(2)
|
||||
)
|
||||
k_conv_weights = self.k_conv1d.weight.view(
|
||||
self.k_conv1d.weight.size(0), self.k_conv1d.weight.size(2)
|
||||
)
|
||||
v_conv_weights = self.v_conv1d.weight.view(
|
||||
self.v_conv1d.weight.size(0), self.v_conv1d.weight.size(2)
|
||||
)
|
||||
if attn_metadata_narrowed.num_prefills > 0:
|
||||
q_proj_states = q_proj_states.transpose(0, 1)
|
||||
k_proj_states = k_proj_states.transpose(0, 1)
|
||||
v_proj_states = v_proj_states.transpose(0, 1)
|
||||
q = causal_conv1d_fn(
|
||||
q_proj_states,
|
||||
q_conv_weights,
|
||||
self.q_conv1d.bias,
|
||||
activation="silu",
|
||||
conv_states=conv_state_q,
|
||||
has_initial_state=has_initial_state,
|
||||
cache_indices=non_spec_state_indices_tensor,
|
||||
query_start_loc=non_spec_query_start_loc,
|
||||
metadata=attn_metadata_narrowed,
|
||||
).transpose(0, 1)
|
||||
k = causal_conv1d_fn(
|
||||
k_proj_states,
|
||||
k_conv_weights,
|
||||
self.k_conv1d.bias,
|
||||
activation="silu",
|
||||
conv_states=conv_state_k,
|
||||
has_initial_state=has_initial_state,
|
||||
cache_indices=non_spec_state_indices_tensor,
|
||||
query_start_loc=non_spec_query_start_loc,
|
||||
metadata=attn_metadata_narrowed,
|
||||
).transpose(0, 1)
|
||||
v = causal_conv1d_fn(
|
||||
v_proj_states,
|
||||
v_conv_weights,
|
||||
self.v_conv1d.bias,
|
||||
activation="silu",
|
||||
conv_states=conv_state_v,
|
||||
has_initial_state=has_initial_state,
|
||||
cache_indices=non_spec_state_indices_tensor,
|
||||
query_start_loc=non_spec_query_start_loc,
|
||||
metadata=attn_metadata_narrowed,
|
||||
).transpose(0, 1)
|
||||
# Split tokens into the multi-query spec-decode part and the remaining
|
||||
# (prefill / plain decode) part.
|
||||
if spec_sequence_masks is not None:
|
||||
if m.num_prefills == 0 and m.num_decodes == 0:
|
||||
mixed_qkv_spec = mixed_qkv
|
||||
g1_spec, beta_spec = g1, beta
|
||||
mixed_qkv_ns = g1_ns = beta_ns = None
|
||||
else:
|
||||
mixed_qkv_spec = mixed_qkv.index_select(0, spec_token_indx)
|
||||
g1_spec = g1.index_select(1, spec_token_indx)
|
||||
beta_spec = beta.index_select(1, spec_token_indx)
|
||||
mixed_qkv_ns = mixed_qkv.index_select(0, non_spec_token_indx)
|
||||
g1_ns = g1.index_select(1, non_spec_token_indx)
|
||||
beta_ns = beta.index_select(1, non_spec_token_indx)
|
||||
else:
|
||||
assert non_spec_state_indices_tensor is not None
|
||||
decode_conv_indices = non_spec_state_indices_tensor[
|
||||
: attn_metadata_narrowed.num_actual_tokens
|
||||
]
|
||||
q = causal_conv1d_update(
|
||||
q_proj_states,
|
||||
conv_state_q,
|
||||
q_conv_weights,
|
||||
self.q_conv1d.bias,
|
||||
activation="silu",
|
||||
conv_state_indices=decode_conv_indices,
|
||||
validate_data=True,
|
||||
)
|
||||
k = causal_conv1d_update(
|
||||
k_proj_states,
|
||||
conv_state_k,
|
||||
k_conv_weights,
|
||||
self.k_conv1d.bias,
|
||||
activation="silu",
|
||||
conv_state_indices=decode_conv_indices,
|
||||
validate_data=True,
|
||||
)
|
||||
v = causal_conv1d_update(
|
||||
v_proj_states,
|
||||
conv_state_v,
|
||||
v_conv_weights,
|
||||
self.v_conv1d.bias,
|
||||
activation="silu",
|
||||
conv_state_indices=decode_conv_indices,
|
||||
validate_data=True,
|
||||
)
|
||||
mixed_qkv_spec = g1_spec = beta_spec = None
|
||||
mixed_qkv_ns, g1_ns, beta_ns = mixed_qkv, g1, beta
|
||||
|
||||
q, k, v = map(
|
||||
lambda x: rearrange(x, "n (h d) -> 1 n h d", d=self.head_dim), (q, k, v)
|
||||
)
|
||||
# ---------- spec-decode multi-query path ----------
|
||||
core_attn_out_spec = None
|
||||
if spec_sequence_masks is not None:
|
||||
assert spec_state_indices_tensor is not None
|
||||
assert spec_query_start_loc is not None
|
||||
spec_conv_indices = spec_state_indices_tensor[:, 0][: m.num_spec_decodes]
|
||||
spec_max_query_len = spec_state_indices_tensor.size(-1)
|
||||
|
||||
if attn_metadata_narrowed.num_prefills > 0:
|
||||
assert non_spec_state_indices_tensor is not None
|
||||
assert has_initial_state is not None
|
||||
zero_idx = non_spec_state_indices_tensor[~has_initial_state]
|
||||
recurrent_state[zero_idx] = 0
|
||||
initial_state = recurrent_state[non_spec_state_indices_tensor].contiguous()
|
||||
(
|
||||
core_attn_out_non_spec,
|
||||
last_recurrent_state,
|
||||
) = chunk_kda_with_fused_gate(
|
||||
q=q,
|
||||
k=k,
|
||||
v=v,
|
||||
raw_g=g1,
|
||||
beta=beta,
|
||||
# Sibling beta and, for full-rank gates, output-gate views remain
|
||||
# live, so write the convolution output separately.
|
||||
spec_conv_out = torch.empty(
|
||||
mixed_qkv_spec.shape,
|
||||
dtype=mixed_qkv_spec.dtype,
|
||||
device=mixed_qkv_spec.device,
|
||||
)
|
||||
mixed_qkv_spec = causal_conv1d_update(
|
||||
mixed_qkv_spec,
|
||||
conv_state,
|
||||
conv_weights,
|
||||
self.conv1d.bias,
|
||||
activation="silu",
|
||||
conv_state_indices=spec_conv_indices,
|
||||
num_accepted_tokens=num_accepted_tokens,
|
||||
query_start_loc=spec_query_start_loc,
|
||||
max_query_len=spec_max_query_len,
|
||||
validate_data=False,
|
||||
out=spec_conv_out,
|
||||
)
|
||||
q_spec, k_spec, v_spec = (
|
||||
rearrange(x, "n (h d) -> 1 n h d", d=self.head_dim)
|
||||
for x in mixed_qkv_spec.split(self.local_projection_size, dim=-1)
|
||||
)
|
||||
spec_cu_seqlens = spec_query_start_loc[: m.num_spec_decodes + 1]
|
||||
# Spec-only batches write directly into core_attn_out.
|
||||
spec_out = (
|
||||
core_attn_out[:, : q_spec.shape[1]]
|
||||
if m.num_prefills == 0 and m.num_decodes == 0
|
||||
else None
|
||||
)
|
||||
core_attn_out_spec, _ = fused_recurrent_kda(
|
||||
q=q_spec,
|
||||
k=k_spec,
|
||||
v=v_spec,
|
||||
raw_g=g1_spec,
|
||||
raw_beta=beta_spec,
|
||||
A_log=self.A_log,
|
||||
g_bias=self.dt_bias,
|
||||
initial_state=initial_state,
|
||||
output_final_state=True,
|
||||
use_qk_l2norm_in_kernel=True,
|
||||
cu_seqlens=non_spec_query_start_loc,
|
||||
)
|
||||
# Init cache
|
||||
recurrent_state[non_spec_state_indices_tensor] = last_recurrent_state
|
||||
else:
|
||||
assert non_spec_query_start_loc is not None
|
||||
g1 = fused_kda_gate(
|
||||
rearrange(g1, "1 n h d -> n (h d)"),
|
||||
self.A_log,
|
||||
self.head_dim,
|
||||
g_bias=self.dt_bias,
|
||||
).unsqueeze(0)
|
||||
(
|
||||
core_attn_out_non_spec,
|
||||
last_recurrent_state,
|
||||
) = fused_recurrent_kda(
|
||||
q=q,
|
||||
k=k,
|
||||
v=v,
|
||||
g=g1,
|
||||
beta=beta,
|
||||
dt_bias=self.dt_bias,
|
||||
lower_bound=self.gate_lower_bound,
|
||||
initial_state=recurrent_state,
|
||||
use_qk_l2norm_in_kernel=True,
|
||||
cu_seqlens=non_spec_query_start_loc[
|
||||
: attn_metadata_narrowed.num_decodes + 1
|
||||
],
|
||||
ssm_state_indices=non_spec_state_indices_tensor,
|
||||
cu_seqlens=spec_cu_seqlens,
|
||||
ssm_state_indices=spec_state_indices_tensor,
|
||||
num_accepted_tokens=num_accepted_tokens,
|
||||
out=spec_out,
|
||||
)
|
||||
core_attn_out[0, :num_actual_tokens] = core_attn_out_non_spec[
|
||||
0, :num_actual_tokens
|
||||
]
|
||||
|
||||
# ---------- non-spec path (prefill or plain decode) ----------
|
||||
core_attn_out_non_spec = None
|
||||
if mixed_qkv_ns is not None:
|
||||
assert g1_ns is not None and beta_ns is not None
|
||||
if m.num_prefills > 0:
|
||||
q_ns, k_ns, v_ns = mixed_qkv_ns.split(
|
||||
self.local_projection_size, dim=-1
|
||||
)
|
||||
|
||||
# Packed prefill conv would require copying V solely to make
|
||||
# it dense for KDA. Separate calls accept the strided inputs
|
||||
# and produce dense Q/K/V without that extra traffic.
|
||||
# TODO: Use packed conv once every KDA prefill backend accepts
|
||||
# row-strided Q/K/V directly.
|
||||
def _prefill_conv(
|
||||
x: torch.Tensor,
|
||||
state: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
return causal_conv1d_fn(
|
||||
x.transpose(0, 1),
|
||||
weight,
|
||||
None,
|
||||
activation="silu",
|
||||
conv_states=state,
|
||||
has_initial_state=has_initial_state,
|
||||
cache_indices=non_spec_state_indices_tensor,
|
||||
query_start_loc=non_spec_query_start_loc,
|
||||
metadata=m,
|
||||
).transpose(0, 1)
|
||||
|
||||
q_ns = _prefill_conv(q_ns, q_conv_state, q_conv_weight)
|
||||
k_ns = _prefill_conv(k_ns, k_conv_state, k_conv_weight)
|
||||
v_ns = _prefill_conv(v_ns, v_conv_state, v_conv_weight)
|
||||
q_ns, k_ns, v_ns = (
|
||||
rearrange(x, "n (h d) -> 1 n h d", d=self.head_dim)
|
||||
for x in (q_ns, k_ns, v_ns)
|
||||
)
|
||||
|
||||
assert non_spec_state_indices_tensor is not None
|
||||
assert has_initial_state is not None
|
||||
initial_state = gather_initial_states(
|
||||
recurrent_state,
|
||||
non_spec_state_indices_tensor,
|
||||
has_initial_state,
|
||||
)
|
||||
(
|
||||
core_attn_out_non_spec,
|
||||
last_recurrent_state,
|
||||
) = chunk_kda_with_fused_gate(
|
||||
q=q_ns,
|
||||
k=k_ns,
|
||||
v=v_ns,
|
||||
raw_g=g1_ns,
|
||||
raw_beta=beta_ns,
|
||||
A_log=self.A_log,
|
||||
g_bias=self.dt_bias,
|
||||
lower_bound=self.gate_lower_bound,
|
||||
initial_state=initial_state,
|
||||
output_final_state=True,
|
||||
use_qk_l2norm_in_kernel=True,
|
||||
cu_seqlens=non_spec_query_start_loc,
|
||||
)
|
||||
# Init cache
|
||||
recurrent_state[non_spec_state_indices_tensor] = last_recurrent_state
|
||||
|
||||
else:
|
||||
# pure-decode non-spec batch
|
||||
assert non_spec_state_indices_tensor is not None
|
||||
decode_conv_indices = non_spec_state_indices_tensor[
|
||||
: mixed_qkv_ns.size(0)
|
||||
]
|
||||
# Sibling beta and, for full-rank gates, output-gate views
|
||||
# remain live, so write the conv output separately.
|
||||
packed_conv_out = torch.empty(
|
||||
mixed_qkv_ns.shape,
|
||||
dtype=mixed_qkv_ns.dtype,
|
||||
device=mixed_qkv_ns.device,
|
||||
)
|
||||
mixed_qkv_ns = causal_conv1d_update(
|
||||
mixed_qkv_ns,
|
||||
conv_state,
|
||||
conv_weights,
|
||||
self.conv1d.bias,
|
||||
activation="silu",
|
||||
conv_state_indices=decode_conv_indices,
|
||||
validate_data=True,
|
||||
out=packed_conv_out,
|
||||
)
|
||||
core_attn_out_non_spec, _ = fused_recurrent_kda_packed_decode(
|
||||
mixed_qkv=mixed_qkv_ns,
|
||||
raw_g=g1_ns,
|
||||
raw_beta=beta_ns,
|
||||
A_log=self.A_log,
|
||||
dt_bias=self.dt_bias,
|
||||
lower_bound=self.gate_lower_bound,
|
||||
initial_state=recurrent_state,
|
||||
state_indices=decode_conv_indices,
|
||||
)
|
||||
|
||||
# ---------- merge spec and non-spec outputs ----------
|
||||
if core_attn_out_spec is not None and core_attn_out_non_spec is not None:
|
||||
# Mixed batches require indexed placement in the original order.
|
||||
merged = torch.empty(
|
||||
(1, num_actual_tokens, *core_attn_out_spec.shape[2:]),
|
||||
dtype=core_attn_out_spec.dtype,
|
||||
device=core_attn_out_spec.device,
|
||||
)
|
||||
merged.index_copy_(1, spec_token_indx, core_attn_out_spec)
|
||||
merged.index_copy_(1, non_spec_token_indx, core_attn_out_non_spec)
|
||||
core_attn_out[0, :num_actual_tokens] = merged[0, :num_actual_tokens]
|
||||
elif core_attn_out_non_spec is not None:
|
||||
core_attn_out[0, :num_actual_tokens] = core_attn_out_non_spec[
|
||||
0, :num_actual_tokens
|
||||
]
|
||||
else:
|
||||
assert core_attn_out_spec is not None
|
||||
core_attn_out.copy_(self.o_norm(core_attn_out, g2))
|
||||
|
||||
@@ -288,7 +288,7 @@ class MambaStateShapeCalculator:
|
||||
|
||||
conv_dim = proj_size + 2 * proj_k_size
|
||||
conv_state_shape = cls._orient_conv_shape(
|
||||
divide(conv_dim, tp_world_size), conv_kernel_size - 1
|
||||
divide(conv_dim, tp_world_size), conv_kernel_size - 1 + num_spec
|
||||
)
|
||||
recurrent_state_shape = (divide(num_heads, tp_world_size), head_dim, head_dim)
|
||||
return (conv_state_shape, recurrent_state_shape)
|
||||
|
||||
@@ -8,11 +8,12 @@
|
||||
import numpy as np
|
||||
import torch
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.v1.attention.backends.utils import NULL_BLOCK_ID, PAD_SLOT_ID
|
||||
|
||||
|
||||
@triton.jit()
|
||||
@triton.jit(do_not_specialize_on_alignment=["num_cache_lines"])
|
||||
def _causal_conv1d_fwd_kernel( # continuous batching
|
||||
# Pointers to matrices
|
||||
x_ptr, # (dim, cu_seqlen) holding `batch` of actual sequences + padded sequences
|
||||
@@ -33,7 +34,7 @@ def _causal_conv1d_fwd_kernel( # continuous batching
|
||||
o_ptr, # (dim, seqlen) - actually pointing to x_ptr
|
||||
# Matrix dimensions
|
||||
dim: tl.constexpr,
|
||||
num_cache_lines: tl.constexpr, # added to support vLLM larger cache lines
|
||||
num_cache_lines, # added to support vLLM larger cache lines
|
||||
# Strides
|
||||
stride_x_dim: tl.constexpr, # stride to get to next feature-value,
|
||||
stride_x_token: tl.int64, # stride to get to next token (same feature-index, same sequence-index)
|
||||
@@ -58,6 +59,7 @@ def _causal_conv1d_fwd_kernel( # continuous batching
|
||||
NP2_STATELEN: tl.constexpr,
|
||||
BLOCK_M: tl.constexpr,
|
||||
BLOCK_N: tl.constexpr,
|
||||
launch_pdl: tl.constexpr,
|
||||
):
|
||||
conv_states_ptr = initial_states_ptr
|
||||
conv_state_indices_ptr = cache_indices_ptr
|
||||
@@ -68,6 +70,9 @@ def _causal_conv1d_fwd_kernel( # continuous batching
|
||||
KERNEL_WIDTH - 1
|
||||
) # can be passed via argument if it's not the same as this value
|
||||
|
||||
if launch_pdl:
|
||||
tl.extra.cuda.gdc_wait()
|
||||
|
||||
# one program handles one chunk in a single sequence
|
||||
# rather than mixing sequences - to make updating initial_states across sequences efficiently
|
||||
|
||||
@@ -79,6 +84,8 @@ def _causal_conv1d_fwd_kernel( # continuous batching
|
||||
idx_feats = tl.program_id(1) * BLOCK_N + tl.arange(0, BLOCK_N)
|
||||
|
||||
if idx_seq == pad_slot_id:
|
||||
if launch_pdl:
|
||||
tl.extra.cuda.gdc_launch_dependents()
|
||||
return
|
||||
|
||||
sequence_start_index = tl.load(query_start_loc_ptr + idx_seq)
|
||||
@@ -136,6 +143,8 @@ def _causal_conv1d_fwd_kernel( # continuous batching
|
||||
if HAS_NULL_BLOCK: # noqa
|
||||
if conv_states_input_coord == null_block_id:
|
||||
# not processing as this is a null block (padding)
|
||||
if launch_pdl:
|
||||
tl.extra.cuda.gdc_launch_dependents()
|
||||
return
|
||||
conv_states_base = (
|
||||
conv_states_ptr
|
||||
@@ -408,6 +417,10 @@ def _causal_conv1d_fwd_kernel( # continuous batching
|
||||
w_ptrs = w_base + (3 * stride_w_width) # [BLOCK_N] tensor
|
||||
w_col3 = tl.load(w_ptrs, mask_w, other=0.0)
|
||||
mask_x_1d = idx_feats < dim
|
||||
|
||||
if launch_pdl:
|
||||
tl.extra.cuda.gdc_launch_dependents()
|
||||
|
||||
for idx_token in range(segment_len):
|
||||
acc = acc_preload
|
||||
|
||||
@@ -741,11 +754,12 @@ def causal_conv1d_fn(
|
||||
BLOCK_M=BLOCK_M,
|
||||
BLOCK_N=256,
|
||||
num_stages=2,
|
||||
launch_pdl=current_platform.is_arch_support_pdl(),
|
||||
)
|
||||
return out.to(original_x_dtype)
|
||||
|
||||
|
||||
@triton.jit()
|
||||
@triton.jit(do_not_specialize_on_alignment=["num_cache_lines"])
|
||||
def _causal_conv1d_update_kernel(
|
||||
# Pointers to matrices
|
||||
x_ptr, # (batch, dim, seqlen)
|
||||
@@ -763,7 +777,7 @@ def _causal_conv1d_update_kernel(
|
||||
dim: tl.constexpr,
|
||||
seqlen: tl.constexpr,
|
||||
state_len: tl.constexpr,
|
||||
num_cache_lines: tl.constexpr, # added to support vLLM larger cache lines
|
||||
num_cache_lines, # added to support vLLM larger cache lines
|
||||
# Strides
|
||||
stride_x_seq: tl.constexpr,
|
||||
stride_x_dim: tl.constexpr,
|
||||
@@ -789,10 +803,16 @@ def _causal_conv1d_update_kernel(
|
||||
NP2_STATELEN: tl.constexpr,
|
||||
HAS_NULL_BLOCK: tl.constexpr,
|
||||
BLOCK_N: tl.constexpr,
|
||||
launch_pdl: tl.constexpr,
|
||||
):
|
||||
if launch_pdl:
|
||||
tl.extra.cuda.gdc_wait()
|
||||
|
||||
# ruff: noqa: E501
|
||||
idx_seq = tl.program_id(0)
|
||||
if idx_seq >= batch:
|
||||
if launch_pdl:
|
||||
tl.extra.cuda.gdc_launch_dependents()
|
||||
return
|
||||
|
||||
# [BLOCK_N,] elements along the feature-dimension (channel)
|
||||
@@ -814,6 +834,8 @@ def _causal_conv1d_update_kernel(
|
||||
if HAS_NULL_BLOCK: # noqa
|
||||
if conv_states_input_coord == null_block_id:
|
||||
# not processing as this is not the actual sequence
|
||||
if launch_pdl:
|
||||
tl.extra.cuda.gdc_launch_dependents()
|
||||
return
|
||||
|
||||
if IS_VARLEN:
|
||||
@@ -831,6 +853,8 @@ def _causal_conv1d_update_kernel(
|
||||
o_offset = idx_seq * stride_o_seq
|
||||
|
||||
if query_start_index == query_end_index:
|
||||
if launch_pdl:
|
||||
tl.extra.cuda.gdc_launch_dependents()
|
||||
return
|
||||
|
||||
if IS_SPEC_DECODING:
|
||||
@@ -969,6 +993,9 @@ def _causal_conv1d_update_kernel(
|
||||
mask_x_1d = idx_feats < dim
|
||||
|
||||
# STEP 5: compute each token
|
||||
if launch_pdl:
|
||||
tl.extra.cuda.gdc_launch_dependents()
|
||||
|
||||
for idx_token in tl.range(seqlen):
|
||||
acc = acc_preload
|
||||
|
||||
@@ -1245,14 +1272,13 @@ def causal_conv1d_update(
|
||||
NP2_STATELEN=np2_statelen,
|
||||
HAS_NULL_BLOCK=null_block_id is not None,
|
||||
BLOCK_N=256,
|
||||
launch_pdl=current_platform.is_arch_support_pdl(),
|
||||
)
|
||||
if unsqueeze:
|
||||
out = out.squeeze(-1)
|
||||
return out.to(original_x_dtype)
|
||||
|
||||
|
||||
from vllm.platforms import current_platform # noqa: E402
|
||||
|
||||
if current_platform.is_cpu():
|
||||
from vllm.model_executor.layers.mamba.ops.cpu.causal_conv1d import (
|
||||
causal_conv1d_fn_cpu,
|
||||
|
||||
@@ -28,6 +28,7 @@ class MLAModules:
|
||||
is_sparse: bool
|
||||
topk_indices_buffer: torch.Tensor | None
|
||||
indexer_rotary_emb: torch.nn.Module | None = None
|
||||
g_proj: torch.nn.Module | None = None
|
||||
|
||||
|
||||
# --8<-- [start:multi_head_latent_attention]
|
||||
@@ -66,6 +67,7 @@ class MultiHeadLatentAttentionWrapper(PluggableLayer):
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
prefix: str = "",
|
||||
skip_topk: bool = False,
|
||||
non_causal_multi_token_decode: bool = False,
|
||||
allow_short_prefill_indexer_scoring_skip: bool = False,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
@@ -89,6 +91,7 @@ class MultiHeadLatentAttentionWrapper(PluggableLayer):
|
||||
self.indexer = mla_modules.indexer
|
||||
self.indexer_rope_emb = mla_modules.indexer_rotary_emb
|
||||
self.is_sparse = mla_modules.is_sparse
|
||||
self.g_proj = mla_modules.g_proj
|
||||
|
||||
# Whether to skip top-k token selection computation in this layer.
|
||||
# When True, the indexer will not be called, and the layer will reuse
|
||||
@@ -120,6 +123,7 @@ class MultiHeadLatentAttentionWrapper(PluggableLayer):
|
||||
use_sparse=self.is_sparse,
|
||||
indexer=self.indexer,
|
||||
topk_indices_buffer=mla_modules.topk_indices_buffer,
|
||||
non_causal_multi_token_decode=non_causal_multi_token_decode,
|
||||
)
|
||||
indexer_op = getattr(self.indexer, "indexer_op", None)
|
||||
if indexer_op is not None and hasattr(
|
||||
@@ -216,4 +220,7 @@ class MultiHeadLatentAttentionWrapper(PluggableLayer):
|
||||
q_dcp_replicated=q_dcp_replicated,
|
||||
)
|
||||
|
||||
if self.g_proj is not None:
|
||||
attn_out = attn_out * self.g_proj(hidden_states)[0].sigmoid()
|
||||
|
||||
return self.o_proj(attn_out)[0]
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
|
||||
import torch
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.attention import Attention
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
@@ -17,6 +18,7 @@ from vllm.model_executor.layers.fused_moe import modular_kernel as mk
|
||||
from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import (
|
||||
TRITON_BACKENDS,
|
||||
Mxfp4MoeBackend,
|
||||
backend_to_kernel_cls,
|
||||
convert_gpt_oss_weight_to_mxfp4_moe_kernel_format,
|
||||
convert_weight_to_mxfp4_moe_kernel_format,
|
||||
make_mxfp4_moe_kernel,
|
||||
@@ -68,8 +70,11 @@ class Mxfp4Config(QuantizationConfig):
|
||||
def get_config_filenames(cls) -> list[str]:
|
||||
return []
|
||||
|
||||
# TODO (zyongye) This is only temporaty fallback.
|
||||
# We should have `Mxfp4MoEMethod` after this migration is complete.
|
||||
def _make_moe_method(self, moe: FusedMoEConfig) -> FusedMoEMethodBase:
|
||||
"""MoE method for RoutedExperts. Subclasses override to pick a
|
||||
checkpoint-specific kernel family."""
|
||||
return Mxfp4MoEMethod(moe)
|
||||
|
||||
def get_quant_method(
|
||||
self, layer: torch.nn.Module, prefix: str
|
||||
) -> "QuantizeMethodBase | None":
|
||||
@@ -86,7 +91,7 @@ class Mxfp4Config(QuantizationConfig):
|
||||
)
|
||||
return UnquantizedLinearMethod()
|
||||
elif isinstance(layer, RoutedExperts):
|
||||
return GptOssMxfp4MoEMethod(layer.moe_config)
|
||||
return self._make_moe_method(layer.moe_config)
|
||||
elif isinstance(layer, Attention):
|
||||
logger.debug_once(
|
||||
"MXFP4 attention layer is not implemented. "
|
||||
@@ -131,6 +136,9 @@ class GptOssMxfp4Config(Mxfp4Config):
|
||||
return None
|
||||
return "gpt_oss_mxfp4"
|
||||
|
||||
def _make_moe_method(self, moe: FusedMoEConfig) -> FusedMoEMethodBase:
|
||||
return GptOssMxfp4MoEMethod(moe)
|
||||
|
||||
|
||||
class GptOssMxfp4MoEMethod(FusedMoEMethodBase):
|
||||
"""MXFP4 MoE quantization method."""
|
||||
@@ -475,13 +483,45 @@ class GptOssMxfp4MoEMethod(FusedMoEMethodBase):
|
||||
)
|
||||
|
||||
|
||||
def _use_k3_situ_aiter(moe: FusedMoEConfig) -> bool:
|
||||
"""Whether Kimi-K3's SiTU MXFP4 MoE should use the AITER A16W4 kernel.
|
||||
|
||||
K3 is weight-only MXFP4 (W4A16) with SiTU activation, which the generic
|
||||
MXFP4 backend selector does not cover; route it to AITER on gfx950.
|
||||
"""
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
if not current_platform.is_rocm():
|
||||
return False
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
from vllm.platforms.rocm import on_gfx950
|
||||
|
||||
return (
|
||||
rocm_aiter_ops.is_fused_moe_enabled()
|
||||
and on_gfx950()
|
||||
and moe.activation == MoEActivation.SITU
|
||||
and moe.activation_situ_linear_beta is not None
|
||||
and rocm_aiter_ops.get_aiter_activation_type("situ") is not None
|
||||
)
|
||||
|
||||
|
||||
class Mxfp4MoEMethod(FusedMoEMethodBase):
|
||||
"""MXFP4 MoE quantization method."""
|
||||
|
||||
def __init__(self, moe: FusedMoEConfig):
|
||||
super().__init__(moe)
|
||||
self.weight_dtype = "mxfp4"
|
||||
self.mxfp4_backend, self.experts_cls = select_deepseek_v4_mxfp4_moe_backend(moe)
|
||||
self.is_k3_situ_aiter = _use_k3_situ_aiter(moe)
|
||||
self.experts_cls: type[mk.FusedMoEExperts] | None
|
||||
if self.is_k3_situ_aiter:
|
||||
self.mxfp4_backend = Mxfp4MoeBackend.AITER_MXFP4_BF16
|
||||
self.experts_cls = backend_to_kernel_cls(self.mxfp4_backend)[0]
|
||||
logger.info_once("Using AITER_MXFP4_BF16 for Kimi-K3 SiTU MXFP4 MoE.")
|
||||
else:
|
||||
self.mxfp4_backend, self.experts_cls = select_deepseek_v4_mxfp4_moe_backend(
|
||||
moe
|
||||
)
|
||||
|
||||
self.max_capture_size = moe.max_capture_size
|
||||
|
||||
@@ -523,6 +563,11 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
|
||||
act_dtype=act_dtype,
|
||||
moe_parallel_config=moe_parallel_config,
|
||||
)
|
||||
if self.is_k3_situ_aiter:
|
||||
# K3's AITER A16W4 kernel handles K3's native intermediate size
|
||||
# (moe_intermediate 3072; e.g. 384/partition at TP8); the generic
|
||||
# 256 round-up would inflate weights and OOM.
|
||||
return hidden_size, intermediate_size_per_partition
|
||||
return mxfp4_round_up_hidden_size_and_intermediate_size(
|
||||
self.mxfp4_backend, hidden_size, intermediate_size_per_partition
|
||||
)
|
||||
@@ -724,7 +769,58 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
|
||||
layer=layer,
|
||||
)
|
||||
|
||||
def _setup_kernel_k3_situ(self, layer: RoutedExperts) -> None:
|
||||
# K3's AITER A16W4 kernel wants the separated ([gate_all, up_all])
|
||||
# stage-1 layout, unlike the interleaved gpt-oss/DeepSeek path in
|
||||
# convert_weight_to_mxfp4_moe_kernel_format. Preshuffle once here.
|
||||
from aiter.utility.fp4_utils import e8m0_shuffle
|
||||
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
|
||||
fp4_dtype = torch.float4_e2m1fn_x2
|
||||
e8m0_dtype = torch.float8_e8m0fnu
|
||||
num_experts = layer.w13_weight.shape[0]
|
||||
|
||||
# a8w4 (AITER_SITUV2_A8W4=1) uses the gate/up-interleaved (_gui_) fp8
|
||||
# flydsl kernels, which need w13 weight+scale in interleave layout.
|
||||
# Default a16w4 keeps the separated layout.
|
||||
guinterleave = envs.AITER_SITUV2_A8W4
|
||||
w13 = rocm_aiter_ops.shuffle_weight_a16w4(
|
||||
layer.w13_weight.data.view(fp4_dtype), 16, guinterleave
|
||||
)
|
||||
w2 = rocm_aiter_ops.shuffle_weight_a16w4(
|
||||
layer.w2_weight.data.view(fp4_dtype), 16, False
|
||||
)
|
||||
w13_scale_raw = layer.w13_weight_scale.data.view(e8m0_dtype)
|
||||
w2_scale_raw = layer.w2_weight_scale.data.view(e8m0_dtype)
|
||||
w13_scale = rocm_aiter_ops.shuffle_scale_a16w4(
|
||||
w13_scale_raw.view(-1, w13_scale_raw.shape[-1]), num_experts, guinterleave
|
||||
)
|
||||
w2_scale = e8m0_shuffle(w2_scale_raw.view(-1, w2_scale_raw.shape[-1]))
|
||||
|
||||
replace_parameter(layer, "w13_weight", w13)
|
||||
replace_parameter(layer, "w2_weight", w2)
|
||||
replace_parameter(layer, "w13_weight_scale", w13_scale)
|
||||
replace_parameter(layer, "w2_weight_scale", w2_scale)
|
||||
layer.w13_weight.is_shuffled = True
|
||||
layer.w2_weight.is_shuffled = True
|
||||
|
||||
self.moe_quant_config = self.get_fused_moe_quant_config(layer)
|
||||
if self.moe_quant_config is not None and self.experts_cls is not None:
|
||||
self.moe_kernel = make_mxfp4_moe_kernel(
|
||||
moe_quant_config=self.moe_quant_config,
|
||||
moe_config=self.moe,
|
||||
mxfp4_backend=self.mxfp4_backend,
|
||||
experts_cls=self.experts_cls,
|
||||
routing_tables=layer._expert_routing_tables(),
|
||||
layer=layer,
|
||||
)
|
||||
|
||||
def process_weights_after_loading(self, layer):
|
||||
if self.is_k3_situ_aiter:
|
||||
self._setup_kernel_k3_situ(layer)
|
||||
return
|
||||
|
||||
w13 = layer.w13_weight
|
||||
w2 = layer.w2_weight
|
||||
w13_scale = layer.w13_weight_scale
|
||||
@@ -819,4 +915,8 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
|
||||
global_num_experts=layer.global_num_experts,
|
||||
expert_map=layer.expert_map,
|
||||
apply_router_weight_on_input=layer.apply_router_weight_on_input,
|
||||
num_expert_group=layer.num_expert_group,
|
||||
topk_group=layer.topk_group,
|
||||
e_score_correction_bias=layer.e_score_correction_bias,
|
||||
routed_scaling_factor=layer.routed_scaling_factor,
|
||||
)
|
||||
|
||||
@@ -18,20 +18,34 @@ def activation_to_flashinfer_int(activation: MoEActivation) -> int:
|
||||
return activation_to_flashinfer_type(activation).value
|
||||
|
||||
|
||||
def has_flashinfer_situ_activation() -> bool:
|
||||
try:
|
||||
from flashinfer.fused_moe.core import ActivationType
|
||||
except ImportError:
|
||||
return False
|
||||
return hasattr(ActivationType, "Situ")
|
||||
|
||||
|
||||
def activation_to_flashinfer_type(activation: MoEActivation) -> "ActivationType":
|
||||
from flashinfer.fused_moe.core import ActivationType
|
||||
|
||||
if activation == MoEActivation.SITU:
|
||||
situ = getattr(ActivationType, "Situ", None)
|
||||
if situ is None:
|
||||
raise ValueError("The installed FlashInfer does not support SITU")
|
||||
return situ
|
||||
|
||||
# silu and gelu are mapped to their gated versions SwiGLU and GeGLU respectively
|
||||
ACTIVATION_TO_FI_ACTIVATION = {
|
||||
MoEActivation.SILU_NO_MUL: ActivationType.Silu,
|
||||
MoEActivation.GELU_NO_MUL: ActivationType.Gelu,
|
||||
MoEActivation.SILU: ActivationType.Swiglu,
|
||||
# SwiGLU-OAI uses Swiglu; the OAI alpha/beta/clamp come from gemm1_* args.
|
||||
MoEActivation.SWIGLUOAI: ActivationType.Swiglu,
|
||||
MoEActivation.SWIGLUOAI_UNINTERLEAVE: ActivationType.Swiglu,
|
||||
MoEActivation.GELU: ActivationType.Geglu,
|
||||
MoEActivation.GELU_TANH: ActivationType.Geglu,
|
||||
MoEActivation.RELU2_NO_MUL: ActivationType.Relu2,
|
||||
MoEActivation.SWIGLUOAI_UNINTERLEAVE: ActivationType.Swiglu,
|
||||
}
|
||||
return ACTIVATION_TO_FI_ACTIVATION[activation]
|
||||
|
||||
|
||||
@@ -474,7 +474,11 @@ def prepare_moe_fp4_layer_for_marlin(
|
||||
|
||||
group_size = 16 if is_nvfp4 else 32
|
||||
|
||||
e = layer.moe_config.num_experts
|
||||
# Use the per-rank (local) expert count: under expert parallelism the
|
||||
# w13_weight/w2_weight tensors only hold this rank's experts (created with
|
||||
# local_num_experts), whereas moe_config.num_experts is the global count.
|
||||
# With no EP the two are equal, so the non-EP path is unchanged.
|
||||
e = layer.moe_config.num_local_experts
|
||||
k = layer.moe_config.hidden_dim
|
||||
n = layer.moe_config.intermediate_size_per_partition
|
||||
|
||||
|
||||
@@ -41,8 +41,8 @@ def _mxfp8_e4m3_quantize_torch(
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Naive MXFP8 quantization.
|
||||
For each block of 32 elements along the last dimension, compute a
|
||||
shared e8m0 scale (the biased exponent of the block-wise amax)
|
||||
and quantize each element to float8_e4m3fn.
|
||||
shared e8m0 scale that fits the block-wise amax into the finite
|
||||
float8_e4m3fn range, and quantize each element to float8_e4m3fn.
|
||||
|
||||
Returns (quantized_values [same shape, fp8], scales uint8).
|
||||
Scale shape depends on is_sf_swizzled_layout:
|
||||
@@ -58,7 +58,8 @@ def _mxfp8_e4m3_quantize_torch(
|
||||
|
||||
amax = x_blocked.abs().amax(dim=-1)
|
||||
amax = amax.clamp(min=torch.finfo(torch.float32).tiny)
|
||||
scale_biased = torch.floor(torch.log2(amax)) + 127.0
|
||||
fp8_max = torch.finfo(MXFP8_VALUE_DTYPE).max
|
||||
scale_biased = torch.ceil(torch.log2(amax / fp8_max)) + 127.0
|
||||
scale_biased = scale_biased.clamp(0, 254)
|
||||
scales_uint8 = scale_biased.to(torch.uint8)
|
||||
|
||||
|
||||
@@ -16,10 +16,9 @@ import vllm.envs as envs
|
||||
from vllm.config import ModelConfig, VllmConfig, set_current_vllm_config
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.attention import (
|
||||
Attention,
|
||||
MLAAttention,
|
||||
MMEncoderAttention,
|
||||
)
|
||||
from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase
|
||||
from vllm.model_executor.layers.hpc import HpcModule
|
||||
from vllm.model_executor.layers.quantization.base_config import (
|
||||
QuantizationConfig,
|
||||
@@ -122,12 +121,13 @@ def process_weights_after_loading(
|
||||
# the caching allocator, which starves the OS on UMA devices.
|
||||
release_device_memory_under_pressure(target_device)
|
||||
|
||||
# Initialize post-load attention weights for Attention, MLA, and MM encoder.
|
||||
# NOTE: Happens after other modules so we can easily decompress weights.
|
||||
# Initialize post-load attention weights for any attention layer and MM
|
||||
# encoder. NOTE: Happens after other modules so we can easily decompress
|
||||
# weights.
|
||||
for _, module in model.named_modules():
|
||||
if isinstance(
|
||||
module, (Attention, MLAAttention, MMEncoderAttention)
|
||||
) and hasattr(module, "process_weights_after_loading"):
|
||||
if isinstance(module, (AttentionLayerBase, MMEncoderAttention)) and hasattr(
|
||||
module, "process_weights_after_loading"
|
||||
):
|
||||
# TODO(lucas): see if there is a way to unify the signatures
|
||||
# of process_weights_after_loading
|
||||
with device_loading_context(module, target_device):
|
||||
|
||||
@@ -339,6 +339,39 @@ class DeepseekV4ForCausalLMConfig(VerifyAndUpdateConfig):
|
||||
)
|
||||
|
||||
|
||||
class KimiK3ForConditionalGenerationConfig(VerifyAndUpdateConfig):
|
||||
"""Route MXFP4-checkpointed Kimi-K3 MoE experts to the MXFP4 interface.
|
||||
|
||||
Kimi-K3 ships its routed experts as compressed-tensors
|
||||
``mxfp4-pack-quantized`` (``quant_method="compressed-tensors"``), which
|
||||
lands them on ``CompressedTensorsW4A4Mxfp4MoEMethod`` and its narrow kernel
|
||||
selection. Rewriting ``quant_method`` to ``"mxfp4"`` selects ``Mxfp4Config``
|
||||
(hence ``Mxfp4MoEMethod``) with its full backend set, while any non-MXFP4
|
||||
checkpoint is left untouched. Covers both the main model and the MTP draft.
|
||||
|
||||
``model_arch_config.quantization_config`` is a separate dict, snapshotted in
|
||||
``ModelConfig.__init__`` before this hook runs, and it is what
|
||||
``_verify_quantization`` reads when resolving the quant method. Patch it
|
||||
alongside the hf configs so the rewrite lands before resolution; otherwise
|
||||
the main model still resolves to compressed-tensors.
|
||||
"""
|
||||
|
||||
@staticmethod
|
||||
def verify_and_update_model_config(model_config: "ModelConfig") -> None:
|
||||
for cfg in (
|
||||
model_config.hf_config,
|
||||
model_config.hf_text_config,
|
||||
model_config.model_arch_config,
|
||||
):
|
||||
quant_config = getattr(cfg, "quantization_config", None)
|
||||
if (
|
||||
isinstance(quant_config, dict)
|
||||
and quant_config.get("quant_method") == "compressed-tensors"
|
||||
and quant_config.get("format") == "mxfp4-pack-quantized"
|
||||
):
|
||||
quant_config["quant_method"] = "mxfp4"
|
||||
|
||||
|
||||
class GptOssForCausalLMConfig(VerifyAndUpdateConfig):
|
||||
@staticmethod
|
||||
def verify_and_update_model_config(model_config: "ModelConfig") -> None:
|
||||
@@ -884,6 +917,8 @@ MODELS_CONFIG_MAP: dict[str, type[VerifyAndUpdateConfig]] = {
|
||||
"JambaForSequenceClassification": JambaForSequenceClassificationConfig,
|
||||
"JinaForRanking": JinaForRankingConfig,
|
||||
"JinaVLForRanking": JinaVLForSequenceClassificationConfig,
|
||||
"KimiK3ForConditionalGeneration": KimiK3ForConditionalGenerationConfig,
|
||||
"KimiK3MTPModel": KimiK3ForConditionalGenerationConfig,
|
||||
"LlamaBidirectionalForSequenceClassification": LlamaBidirectionalConfig,
|
||||
"LlamaBidirectionalModel": LlamaBidirectionalConfig,
|
||||
"LlamaNemotronVLForSequenceClassification": LlamaNemotronVLConfig,
|
||||
|
||||
@@ -983,6 +983,7 @@ class DeepseekV2MLAAttention(nn.Module):
|
||||
topk_indices_buffer: torch.Tensor | None = None,
|
||||
input_size: int | None = None,
|
||||
reduce_results: bool = True,
|
||||
non_causal_multi_token_decode: bool = False,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.hidden_size = hidden_size
|
||||
@@ -1178,6 +1179,7 @@ class DeepseekV2MLAAttention(nn.Module):
|
||||
# the V1 proposer. A frozen True would leave the draft reading a
|
||||
# never-written topk buffer.
|
||||
skip_topk=_skip_topk and not is_mtp_layer,
|
||||
non_causal_multi_token_decode=non_causal_multi_token_decode,
|
||||
# Do not skip scoring for MTP layers: their top-k buffer may be
|
||||
# reused by later draft iterations through index sharing.
|
||||
allow_short_prefill_indexer_scoring_skip=not is_mtp_layer,
|
||||
|
||||
@@ -1407,7 +1407,7 @@ class EagleModelMixin:
|
||||
aux_hidden_states: list[torch.Tensor],
|
||||
layer_idx: int,
|
||||
hidden_states: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
residual: torch.Tensor | None,
|
||||
) -> list[torch.Tensor]:
|
||||
if layer_idx in self.aux_hidden_state_layers:
|
||||
value = hidden_states + residual if residual is not None else hidden_states
|
||||
|
||||
@@ -34,6 +34,7 @@ from vllm.model_executor.models.vision import (
|
||||
is_vit_use_data_parallel,
|
||||
run_dp_sharded_mrope_vision_model,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.transformers_utils.configs.kimi_k25 import KimiK25VisionConfig
|
||||
|
||||
logger = init_logger(__name__)
|
||||
@@ -154,9 +155,7 @@ class Learnable2DInterpPosEmbDivided_fixed(nn.Module):
|
||||
def reset_parameters(self):
|
||||
nn.init.normal_(self.weight)
|
||||
|
||||
def forward(
|
||||
self, x: torch.Tensor, grid_thws: torch.Tensor | list[list[int]]
|
||||
) -> torch.Tensor:
|
||||
def get_pos_embeds(self, grid_thws: torch.Tensor | list[list[int]]) -> torch.Tensor:
|
||||
pos_embs = []
|
||||
grid_thw_list = grid_thws if isinstance(grid_thws, list) else grid_thws.tolist()
|
||||
for t, h, w in grid_thw_list:
|
||||
@@ -179,8 +178,12 @@ class Learnable2DInterpPosEmbDivided_fixed(nn.Module):
|
||||
|
||||
pos_embs.append(pos_emb_3d.reshape(-1, pos_emb_3d.shape[-1]))
|
||||
|
||||
out = x + torch.cat(pos_embs)
|
||||
return out
|
||||
return torch.cat(pos_embs)
|
||||
|
||||
def forward(
|
||||
self, x: torch.Tensor, grid_thws: torch.Tensor | list[list[int]]
|
||||
) -> torch.Tensor:
|
||||
return x + self.get_pos_embeds(grid_thws)
|
||||
|
||||
|
||||
class MoonVision3dPatchEmbed(nn.Module):
|
||||
@@ -195,6 +198,8 @@ class MoonVision3dPatchEmbed(nn.Module):
|
||||
pos_emb_width: int = 14,
|
||||
pos_emb_time: int = 4,
|
||||
pos_emb_type: str = "divided_fixed",
|
||||
patch_embed_proj_bias: bool = True,
|
||||
pos_emb_interpolation_mode: str = "bicubic",
|
||||
):
|
||||
super().__init__()
|
||||
assert isinstance(patch_size, int | Sequence), (
|
||||
@@ -208,7 +213,11 @@ class MoonVision3dPatchEmbed(nn.Module):
|
||||
self.patch_size = patch_size
|
||||
|
||||
self.proj = nn.Conv2d(
|
||||
in_dim, out_dim, kernel_size=patch_size, stride=patch_size
|
||||
in_dim,
|
||||
out_dim,
|
||||
kernel_size=patch_size,
|
||||
stride=patch_size,
|
||||
bias=patch_embed_proj_bias,
|
||||
)
|
||||
|
||||
if pos_emb_type == "divided_fixed":
|
||||
@@ -217,17 +226,37 @@ class MoonVision3dPatchEmbed(nn.Module):
|
||||
width=pos_emb_width,
|
||||
num_frames=pos_emb_time,
|
||||
dim=out_dim,
|
||||
interpolation_mode=pos_emb_interpolation_mode,
|
||||
)
|
||||
else:
|
||||
raise NotImplementedError(f"Not support pos_emb_type: {pos_emb_type}")
|
||||
|
||||
def forward(
|
||||
self, x: torch.Tensor, grid_thws: torch.Tensor | list[list[int]]
|
||||
self,
|
||||
x: torch.Tensor,
|
||||
grid_thws: torch.Tensor | list[list[int]] | None,
|
||||
*,
|
||||
pos_embeds: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
x = self.proj(x).view(x.size(0), -1)
|
||||
# apply positional embedding
|
||||
x = self.pos_emb(x, grid_thws)
|
||||
return x
|
||||
x = self._proj(x).view(x.size(0), -1)
|
||||
if pos_embeds is not None:
|
||||
return x + pos_embeds
|
||||
assert grid_thws is not None
|
||||
return self.pos_emb(x, grid_thws)
|
||||
|
||||
def _proj(self, x: torch.Tensor) -> torch.Tensor:
|
||||
# MIOpen conv2d intermittently fails under load on ROCm; use aiter Triton.
|
||||
if current_platform.is_rocm() and x.dtype in (torch.float16, torch.bfloat16):
|
||||
from aiter.ops.triton.conv.conv2d import conv2d
|
||||
|
||||
return conv2d(
|
||||
x,
|
||||
self.proj.weight,
|
||||
self.proj.bias,
|
||||
stride=self.patch_size,
|
||||
layout="nchw",
|
||||
)
|
||||
return self.proj(x)
|
||||
|
||||
|
||||
class Rope2DPosEmbRepeated(nn.Module):
|
||||
@@ -342,6 +371,14 @@ class MLP2(nn.Module):
|
||||
return x
|
||||
|
||||
|
||||
def _make_vision_norm(norm_type: str, hidden_dim: int) -> nn.Module:
|
||||
if norm_type == "layernorm":
|
||||
return nn.LayerNorm(hidden_dim)
|
||||
if norm_type == "rmsnorm":
|
||||
return nn.RMSNorm(hidden_dim)
|
||||
raise NotImplementedError(f"Not support norm_type: {norm_type}")
|
||||
|
||||
|
||||
class MoonViTEncoderLayer(nn.Module):
|
||||
"""Single encoder layer for MoonViT with TP/DP support."""
|
||||
|
||||
@@ -355,23 +392,33 @@ class MoonViTEncoderLayer(nn.Module):
|
||||
*,
|
||||
activation=F.gelu,
|
||||
attn_bias: bool = False,
|
||||
qkv_hidden_size: int | None = None,
|
||||
norm_type: str = "layernorm",
|
||||
mlp_type: str = "mlp2",
|
||||
linear_bias: bool = True,
|
||||
):
|
||||
super().__init__()
|
||||
self.use_data_parallel = is_vit_use_data_parallel(num_heads)
|
||||
|
||||
self.num_heads = num_heads
|
||||
self.hidden_dim = hidden_dim
|
||||
self.hidden_size_per_attention_head = self.hidden_dim // self.num_heads
|
||||
self.qkv_hidden_size = (
|
||||
hidden_dim if qkv_hidden_size is None else qkv_hidden_size
|
||||
)
|
||||
self.hidden_size_per_attention_head = self.qkv_hidden_size // self.num_heads
|
||||
self.tp_size = (
|
||||
1 if self.use_data_parallel else get_tensor_model_parallel_world_size()
|
||||
)
|
||||
self.num_attention_heads_per_partition = divide(num_heads, self.tp_size)
|
||||
|
||||
self.norm0 = nn.LayerNorm(hidden_dim)
|
||||
self.norm1 = nn.LayerNorm(hidden_dim)
|
||||
self.norm0 = _make_vision_norm(norm_type, hidden_dim)
|
||||
self.norm1 = _make_vision_norm(norm_type, hidden_dim)
|
||||
if mlp_type != "mlp2":
|
||||
raise NotImplementedError(f"Not support mlp_type: {mlp_type}")
|
||||
self.mlp = MLP2(
|
||||
[hidden_dim, mlp_dim, hidden_dim],
|
||||
activation,
|
||||
bias=linear_bias,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.mlp",
|
||||
use_data_parallel=self.use_data_parallel,
|
||||
@@ -387,7 +434,7 @@ class MoonViTEncoderLayer(nn.Module):
|
||||
disable_tp=self.use_data_parallel,
|
||||
)
|
||||
self.wo = RowParallelLinear(
|
||||
hidden_dim,
|
||||
self.qkv_hidden_size,
|
||||
hidden_dim,
|
||||
bias=attn_bias,
|
||||
quant_config=quant_config,
|
||||
@@ -493,8 +540,9 @@ class MoonViT3dEncoder(nn.Module):
|
||||
f'video_attn_type must be "spatial_temporal", got {video_attn_type}'
|
||||
)
|
||||
self.video_attn_type = video_attn_type
|
||||
qkv_hidden_size = block_cfg.get("qkv_hidden_size") or block_cfg["hidden_dim"]
|
||||
self.rope_2d = Rope2DPosEmbRepeated(
|
||||
block_cfg["hidden_dim"] // block_cfg["num_heads"], 512, 512
|
||||
qkv_hidden_size // block_cfg["num_heads"], 512, 512
|
||||
)
|
||||
self.blocks = nn.ModuleList(
|
||||
[
|
||||
@@ -506,13 +554,17 @@ class MoonViT3dEncoder(nn.Module):
|
||||
for layer_idx in range(num_layers)
|
||||
]
|
||||
)
|
||||
self.final_layernorm = nn.LayerNorm(hidden_dim)
|
||||
self.final_layernorm = _make_vision_norm(
|
||||
block_cfg.get("norm_type", "layernorm"), hidden_dim
|
||||
)
|
||||
|
||||
def prepare_encoder_metadata(
|
||||
self,
|
||||
grid_thw_list: list[list[int]],
|
||||
*,
|
||||
device: torch.device,
|
||||
max_batch_size: int | None = None,
|
||||
max_seqlen_override: int | None = None,
|
||||
) -> dict[str, torch.Tensor | None]:
|
||||
metadata: dict[str, torch.Tensor | None] = {}
|
||||
metadata["rope_freqs_cis"] = self.rope_2d.get_freqs_cis(
|
||||
@@ -524,15 +576,30 @@ class MoonViT3dEncoder(nn.Module):
|
||||
cu_seqlens = np.concatenate(
|
||||
[np.zeros(1, dtype=np.int32), lengths.cumsum(dtype=np.int32)]
|
||||
)
|
||||
if max_batch_size is not None:
|
||||
num_seqs = len(cu_seqlens) - 1
|
||||
if num_seqs < max_batch_size:
|
||||
cu_seqlens = np.concatenate(
|
||||
[
|
||||
cu_seqlens,
|
||||
np.full(
|
||||
max_batch_size - num_seqs,
|
||||
cu_seqlens[-1],
|
||||
dtype=np.int32,
|
||||
),
|
||||
]
|
||||
)
|
||||
|
||||
attn_backend = self.blocks[0].attn.attn_backend
|
||||
metadata["sequence_lengths"] = MMEncoderAttention.maybe_compute_seq_lens(
|
||||
attn_backend, cu_seqlens, device
|
||||
)
|
||||
metadata["max_seqlen"] = torch.tensor(
|
||||
MMEncoderAttention.compute_max_seqlen(attn_backend, cu_seqlens),
|
||||
dtype=torch.int32,
|
||||
max_seqlen = (
|
||||
max_seqlen_override
|
||||
if max_seqlen_override is not None
|
||||
else MMEncoderAttention.compute_max_seqlen(attn_backend, cu_seqlens)
|
||||
)
|
||||
metadata["max_seqlen"] = torch.tensor(max_seqlen, dtype=torch.int32)
|
||||
metadata["cu_seqlens"] = MMEncoderAttention.maybe_recompute_cu_seqlens(
|
||||
attn_backend,
|
||||
cu_seqlens,
|
||||
@@ -545,11 +612,12 @@ class MoonViT3dEncoder(nn.Module):
|
||||
def forward(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
grid_thws: torch.Tensor | list[list[int]],
|
||||
grid_thws: torch.Tensor | list[list[int]] | None,
|
||||
*,
|
||||
encoder_metadata: dict[str, torch.Tensor | None] | None = None,
|
||||
) -> torch.Tensor:
|
||||
if encoder_metadata is None:
|
||||
assert grid_thws is not None
|
||||
grid_thw_list = (
|
||||
grid_thws if isinstance(grid_thws, list) else grid_thws.tolist()
|
||||
)
|
||||
@@ -604,6 +672,34 @@ def tpool_patch_merger(
|
||||
return outputs
|
||||
|
||||
|
||||
def build_image_merge_gather_idx(
|
||||
grid_thws: list[list[int]] | list[tuple[int, int, int]],
|
||||
merge_kernel_size: tuple[int, int],
|
||||
) -> np.ndarray:
|
||||
"""Build packed spatial-merge indices for image-only CUDA graphs."""
|
||||
kh, kw = merge_kernel_size
|
||||
parts: list[np.ndarray] = []
|
||||
offset = 0
|
||||
for t, h, w in grid_thws:
|
||||
if t != 1:
|
||||
raise ValueError("Image encoder CUDA graphs require grid T == 1")
|
||||
idx = np.arange(h * w, dtype=np.int64).reshape(h, w)
|
||||
idx = idx.reshape(h // kh, kh, w // kw, kw)
|
||||
parts.append(idx.transpose(0, 2, 1, 3).reshape(-1, kh * kw) + offset)
|
||||
offset += h * w
|
||||
if not parts:
|
||||
return np.empty((0, kh * kw), dtype=np.int64)
|
||||
return np.concatenate(parts)
|
||||
|
||||
|
||||
def tpool_patch_merger_packed(
|
||||
x: torch.Tensor,
|
||||
merge_gather_idx: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
"""Apply the image-only spatial merge using precomputed tensor indices."""
|
||||
return x[merge_gather_idx]
|
||||
|
||||
|
||||
class MoonViT3dPretrainedModel(nn.Module):
|
||||
"""Main vision tower model.
|
||||
|
||||
@@ -630,6 +726,10 @@ class MoonViT3dPretrainedModel(nn.Module):
|
||||
pos_emb_width=config.init_pos_emb_width,
|
||||
pos_emb_time=config.init_pos_emb_time,
|
||||
pos_emb_type=config.pos_emb_type,
|
||||
patch_embed_proj_bias=getattr(config, "patch_embed_proj_bias", True),
|
||||
pos_emb_interpolation_mode=getattr(
|
||||
config, "pos_emb_interpolation_mode", "bicubic"
|
||||
),
|
||||
)
|
||||
|
||||
self.encoder = MoonViT3dEncoder(
|
||||
@@ -638,9 +738,15 @@ class MoonViT3dPretrainedModel(nn.Module):
|
||||
block_cfg={
|
||||
"num_heads": config.num_attention_heads,
|
||||
"hidden_dim": config.hidden_size,
|
||||
"qkv_hidden_size": getattr(config, "qkv_hidden_size", None),
|
||||
"mlp_dim": config.intermediate_size,
|
||||
"activation": get_act_fn("gelu_pytorch_tanh"),
|
||||
"attn_bias": True,
|
||||
"activation": get_act_fn(
|
||||
getattr(config, "activation_func", "gelu_pytorch_tanh")
|
||||
),
|
||||
"attn_bias": getattr(config, "attn_bias", True),
|
||||
"norm_type": getattr(config, "norm_type", "layernorm"),
|
||||
"mlp_type": getattr(config, "mlp_type", "mlp2"),
|
||||
"linear_bias": getattr(config, "linear_bias", True),
|
||||
},
|
||||
video_attn_type=config.video_attn_type,
|
||||
quant_config=quant_config,
|
||||
@@ -650,7 +756,7 @@ class MoonViT3dPretrainedModel(nn.Module):
|
||||
def forward(
|
||||
self,
|
||||
pixel_values: torch.Tensor,
|
||||
grid_thws: torch.Tensor | list[list[int]],
|
||||
grid_thws: torch.Tensor | list[list[int]] | None,
|
||||
*,
|
||||
encoder_metadata: dict[str, torch.Tensor | None] | None = None,
|
||||
) -> torch.Tensor:
|
||||
@@ -662,6 +768,22 @@ class MoonViT3dPretrainedModel(nn.Module):
|
||||
Returns:
|
||||
torch.Tensor: The output tokens.
|
||||
"""
|
||||
if encoder_metadata is not None and "pos_embeds" in encoder_metadata:
|
||||
hidden_states = self.patch_embed(
|
||||
pixel_values,
|
||||
None,
|
||||
pos_embeds=encoder_metadata["pos_embeds"],
|
||||
)
|
||||
hidden_states = self.encoder(
|
||||
hidden_states,
|
||||
None,
|
||||
encoder_metadata=encoder_metadata,
|
||||
)
|
||||
merge_gather_idx = encoder_metadata["merge_gather_idx"]
|
||||
assert merge_gather_idx is not None
|
||||
return tpool_patch_merger_packed(hidden_states, merge_gather_idx)
|
||||
|
||||
assert grid_thws is not None
|
||||
grid_thw_list = grid_thws if isinstance(grid_thws, list) else grid_thws.tolist()
|
||||
if encoder_metadata is None:
|
||||
encoder_metadata = self.encoder.prepare_encoder_metadata(
|
||||
@@ -685,13 +807,40 @@ class MoonViT3dPretrainedModel(nn.Module):
|
||||
|
||||
return hidden_states
|
||||
|
||||
def prepare_encoder_cudagraph_metadata(
|
||||
self,
|
||||
grid_thw_list: list[list[int]],
|
||||
*,
|
||||
max_batch_size: int,
|
||||
max_seqlen_override: int | None = None,
|
||||
device: torch.device,
|
||||
) -> dict[str, torch.Tensor | None]:
|
||||
"""Precompute fixed-buffer metadata for image encoder CUDA graphs."""
|
||||
grid_thw_list = [list(map(int, grid)) for grid in grid_thw_list]
|
||||
metadata = self.encoder.prepare_encoder_metadata(
|
||||
grid_thw_list,
|
||||
device=device,
|
||||
max_batch_size=max_batch_size,
|
||||
max_seqlen_override=max_seqlen_override,
|
||||
)
|
||||
metadata["pos_embeds"] = self.patch_embed.pos_emb.get_pos_embeds(
|
||||
grid_thw_list
|
||||
).to(device=device)
|
||||
merge_gather_idx = build_image_merge_gather_idx(
|
||||
grid_thw_list, self.merge_kernel_size
|
||||
)
|
||||
metadata["merge_gather_idx"] = torch.from_numpy(merge_gather_idx).to(
|
||||
device=device, non_blocking=True
|
||||
)
|
||||
return metadata
|
||||
|
||||
|
||||
@torch.inference_mode()
|
||||
def mm_projector_forward(mm_projector: torch.nn.Module, vt_output: list[torch.Tensor]):
|
||||
"""Apply MM projector to vision tower outputs."""
|
||||
num_embedding_list = [x.shape[0] for x in vt_output]
|
||||
batched = torch.cat(vt_output, dim=0)
|
||||
projector_dtype = mm_projector.pre_norm.weight.dtype
|
||||
projector_dtype = next(mm_projector.parameters()).dtype
|
||||
if batched.dtype != projector_dtype:
|
||||
batched = batched.to(projector_dtype)
|
||||
proj_out = mm_projector(batched)
|
||||
@@ -747,11 +896,34 @@ class KimiK25MultiModalProjector(nn.Module):
|
||||
):
|
||||
super().__init__()
|
||||
self.use_data_parallel = use_data_parallel
|
||||
self.mm_projector_type = getattr(config, "mm_projector_type", "patchmerger")
|
||||
|
||||
# Hidden size after patch merging
|
||||
merge_h, merge_w = config.merge_kernel_size
|
||||
self.hidden_size = config.hidden_size * merge_h * merge_w
|
||||
|
||||
if self.mm_projector_type == "patchmergerv2":
|
||||
self.linear_1 = ReplicatedLinear(
|
||||
self.hidden_size,
|
||||
self.hidden_size,
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.linear_1",
|
||||
)
|
||||
self.linear_2 = ReplicatedLinear(
|
||||
self.hidden_size,
|
||||
getattr(config, "text_hidden_size", config.mm_hidden_size),
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.linear_2",
|
||||
)
|
||||
self.post_norm = torch.nn.RMSNorm(
|
||||
getattr(config, "text_hidden_size", config.mm_hidden_size),
|
||||
eps=config.projector_ln_eps,
|
||||
)
|
||||
self.act = GELUActivation()
|
||||
return
|
||||
|
||||
self.pre_norm = torch.nn.LayerNorm(config.hidden_size, eps=1e-5)
|
||||
self.linear_1 = ReplicatedLinear(
|
||||
self.hidden_size,
|
||||
@@ -770,6 +942,13 @@ class KimiK25MultiModalProjector(nn.Module):
|
||||
self.act = GELUActivation()
|
||||
|
||||
def forward(self, image_features: torch.Tensor) -> torch.Tensor:
|
||||
if self.mm_projector_type == "patchmergerv2":
|
||||
hidden_states = image_features.view(image_features.shape[0], -1)
|
||||
hidden_states, _ = self.linear_1(hidden_states)
|
||||
hidden_states = self.act(hidden_states)
|
||||
hidden_states, _ = self.linear_2(hidden_states)
|
||||
return self.post_norm(hidden_states)
|
||||
|
||||
hidden_states = self.pre_norm(image_features).view(-1, self.hidden_size)
|
||||
hidden_states, _ = self.linear_1(hidden_states)
|
||||
hidden_states = self.act(hidden_states)
|
||||
|
||||
@@ -1,646 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from collections.abc import Iterable
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
from vllm.compilation.decorators import support_torch_compile
|
||||
from vllm.config import CacheConfig, VllmConfig
|
||||
from vllm.distributed import (
|
||||
get_pp_group,
|
||||
get_tensor_model_parallel_world_size,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.activation import SiluAndMul
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoE,
|
||||
fused_moe_make_expert_params_mapping,
|
||||
)
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.model_executor.layers.linear import (
|
||||
ColumnParallelLinear,
|
||||
MergedColumnParallelLinear,
|
||||
ReplicatedLinear,
|
||||
RowParallelLinear,
|
||||
)
|
||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||
from vllm.model_executor.layers.mamba.gdn.kimi_gdn_linear_attn import (
|
||||
KimiGatedDeltaNetAttention,
|
||||
)
|
||||
from vllm.model_executor.layers.mamba.mamba_utils import (
|
||||
MambaStateCopyFunc,
|
||||
MambaStateCopyFuncCalculator,
|
||||
MambaStateDtypeCalculator,
|
||||
MambaStateShapeCalculator,
|
||||
)
|
||||
from vllm.model_executor.layers.mla import MLAModules, MultiHeadLatentAttentionWrapper
|
||||
from vllm.model_executor.layers.quantization.base_config import QuantizationConfig
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
ParallelLMHead,
|
||||
VocabParallelEmbedding,
|
||||
)
|
||||
from vllm.model_executor.model_loader.weight_utils import (
|
||||
default_weight_loader,
|
||||
maybe_remap_kv_scale_name,
|
||||
)
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.transformers_utils.configs.kimi_linear import KimiLinearConfig
|
||||
|
||||
from .interfaces import HasInnerState, IsHybrid, MixtureOfExperts, SupportsPP
|
||||
from .utils import (
|
||||
AutoWeightsLoader,
|
||||
PPMissingLayer,
|
||||
get_spec_layer_idx_from_weight_name,
|
||||
is_pp_missing_parameter,
|
||||
make_layers,
|
||||
maybe_prefix,
|
||||
)
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class KimiMLP(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
hidden_size: int,
|
||||
intermediate_size: int,
|
||||
hidden_act: str,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
reduce_results: bool = True,
|
||||
prefix: str = "",
|
||||
) -> None:
|
||||
super().__init__()
|
||||
|
||||
self.gate_up_proj = MergedColumnParallelLinear(
|
||||
hidden_size,
|
||||
[intermediate_size] * 2,
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.gate_up_proj",
|
||||
)
|
||||
self.down_proj = RowParallelLinear(
|
||||
intermediate_size,
|
||||
hidden_size,
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
reduce_results=reduce_results,
|
||||
prefix=f"{prefix}.down_proj",
|
||||
)
|
||||
if hidden_act != "silu":
|
||||
raise ValueError(
|
||||
f"Unsupported activation: {hidden_act}. Only silu is supported for now."
|
||||
)
|
||||
self.act_fn = SiluAndMul()
|
||||
|
||||
def forward(self, x):
|
||||
gate_up, _ = self.gate_up_proj(x)
|
||||
x = self.act_fn(gate_up)
|
||||
x, _ = self.down_proj(x)
|
||||
return x
|
||||
|
||||
|
||||
class KimiMoE(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
config: KimiLinearConfig,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
prefix: str = "",
|
||||
layer_idx: int = 0,
|
||||
):
|
||||
super().__init__()
|
||||
hidden_size = config.hidden_size
|
||||
intermediate_size = config.intermediate_size
|
||||
moe_intermediate_size = config.moe_intermediate_size
|
||||
num_experts = config.num_experts
|
||||
moe_renormalize = config.moe_renormalize
|
||||
self.tp_size = get_tensor_model_parallel_world_size()
|
||||
self.routed_scaling_factor = config.routed_scaling_factor
|
||||
self.num_shared_experts = config.num_shared_experts
|
||||
self.layer_idx = layer_idx
|
||||
|
||||
if config.hidden_act != "silu":
|
||||
raise ValueError(
|
||||
f"Unsupported activation: {config.hidden_act}. "
|
||||
"Only silu is supported for now."
|
||||
)
|
||||
|
||||
# Gate always runs at half / full precision for now.
|
||||
self.gate = ReplicatedLinear(
|
||||
hidden_size,
|
||||
num_experts,
|
||||
bias=False,
|
||||
quant_config=None,
|
||||
prefix=f"{prefix}.gate",
|
||||
)
|
||||
|
||||
self.gate.e_score_correction_bias = nn.Parameter(torch.empty(num_experts))
|
||||
|
||||
if self.num_shared_experts is not None:
|
||||
intermediate_size = moe_intermediate_size * self.num_shared_experts
|
||||
self.shared_experts = KimiMLP(
|
||||
hidden_size=config.hidden_size,
|
||||
intermediate_size=intermediate_size,
|
||||
hidden_act=config.hidden_act,
|
||||
quant_config=quant_config,
|
||||
reduce_results=False,
|
||||
prefix=f"{prefix}.shared_experts",
|
||||
)
|
||||
else:
|
||||
self.shared_experts = None
|
||||
|
||||
self.experts = FusedMoE(
|
||||
shared_experts=self.shared_experts,
|
||||
num_experts=num_experts,
|
||||
top_k=config.num_experts_per_token,
|
||||
hidden_size=hidden_size,
|
||||
intermediate_size=moe_intermediate_size,
|
||||
renormalize=moe_renormalize,
|
||||
quant_config=quant_config,
|
||||
use_grouped_topk=config.use_grouped_topk,
|
||||
num_expert_group=config.num_expert_group,
|
||||
topk_group=config.topk_group,
|
||||
prefix=f"{prefix}.experts",
|
||||
scoring_func=config.moe_router_activation_func,
|
||||
e_score_correction_bias=self.gate.e_score_correction_bias,
|
||||
routed_scaling_factor=self.routed_scaling_factor,
|
||||
)
|
||||
|
||||
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
||||
num_tokens, hidden_size = hidden_states.shape
|
||||
hidden_states = hidden_states.view(-1, hidden_size)
|
||||
router_logits, _ = self.gate(hidden_states)
|
||||
final_hidden_states = self.experts(
|
||||
hidden_states=hidden_states, router_logits=router_logits
|
||||
)
|
||||
return final_hidden_states.view(num_tokens, hidden_size)
|
||||
|
||||
|
||||
class KimiMLAAttention(nn.Module):
|
||||
"""
|
||||
Main reference: DeepseekV2 vllm Implementation
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: KimiLinearConfig,
|
||||
hidden_size: int,
|
||||
num_heads: int,
|
||||
qk_nope_head_dim: int,
|
||||
qk_rope_head_dim: int,
|
||||
v_head_dim: int,
|
||||
q_lora_rank: int | None,
|
||||
kv_lora_rank: int,
|
||||
use_nope: bool = False,
|
||||
cache_config: CacheConfig | None = None,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
prefix: str = "",
|
||||
**kwargs,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.hidden_size = hidden_size
|
||||
self.qk_nope_head_dim = qk_nope_head_dim
|
||||
self.qk_rope_head_dim = qk_rope_head_dim
|
||||
self.qk_head_dim = qk_nope_head_dim + qk_rope_head_dim
|
||||
self.v_head_dim = v_head_dim
|
||||
self.q_lora_rank = q_lora_rank
|
||||
self.kv_lora_rank = kv_lora_rank
|
||||
self.num_heads = num_heads
|
||||
tp_size = get_tensor_model_parallel_world_size()
|
||||
self.num_local_heads = num_heads // tp_size
|
||||
self.scaling = self.qk_head_dim**-0.5
|
||||
self.use_nope = use_nope
|
||||
assert self.use_nope is True
|
||||
assert self.q_lora_rank is None
|
||||
assert num_heads % tp_size == 0
|
||||
self.kv_a_proj_with_mqa = ReplicatedLinear(
|
||||
self.hidden_size,
|
||||
self.kv_lora_rank + self.qk_rope_head_dim,
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.kv_a_proj_with_mqa",
|
||||
)
|
||||
self.q_proj = ColumnParallelLinear(
|
||||
self.hidden_size,
|
||||
self.num_heads * self.qk_head_dim,
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.q_proj",
|
||||
)
|
||||
self.kv_a_layernorm = RMSNorm(
|
||||
self.kv_lora_rank,
|
||||
eps=config.rms_norm_eps,
|
||||
)
|
||||
self.kv_b_proj = ColumnParallelLinear(
|
||||
self.kv_lora_rank,
|
||||
self.num_heads * (self.qk_nope_head_dim + self.v_head_dim),
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.kv_b_proj",
|
||||
)
|
||||
self.o_proj = RowParallelLinear(
|
||||
self.num_heads * self.v_head_dim,
|
||||
self.hidden_size,
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.o_proj",
|
||||
)
|
||||
|
||||
mla_modules = MLAModules(
|
||||
kv_a_layernorm=self.kv_a_layernorm,
|
||||
kv_b_proj=self.kv_b_proj,
|
||||
rotary_emb=None,
|
||||
o_proj=self.o_proj,
|
||||
fused_qkv_a_proj=None,
|
||||
kv_a_proj_with_mqa=self.kv_a_proj_with_mqa,
|
||||
q_a_layernorm=None,
|
||||
q_b_proj=None,
|
||||
q_proj=self.q_proj,
|
||||
indexer=None,
|
||||
is_sparse=False,
|
||||
topk_indices_buffer=None,
|
||||
)
|
||||
self.mla_attn = MultiHeadLatentAttentionWrapper(
|
||||
self.hidden_size,
|
||||
self.num_local_heads,
|
||||
self.scaling,
|
||||
self.qk_nope_head_dim,
|
||||
self.qk_rope_head_dim,
|
||||
self.v_head_dim,
|
||||
self.q_lora_rank,
|
||||
self.kv_lora_rank,
|
||||
mla_modules,
|
||||
cache_config,
|
||||
quant_config,
|
||||
prefix,
|
||||
)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
positions: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
output: torch.Tensor,
|
||||
) -> None:
|
||||
output[:] = self.mla_attn(positions, hidden_states)
|
||||
|
||||
|
||||
class KimiDecoderLayer(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
config: KimiLinearConfig,
|
||||
vllm_config: VllmConfig,
|
||||
prefix: str = "",
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.hidden_size = config.hidden_size
|
||||
|
||||
self.is_moe = config.is_moe
|
||||
layer_idx = int(prefix.rsplit(".", 1)[1])
|
||||
model_config = vllm_config.model_config
|
||||
cache_config = vllm_config.cache_config
|
||||
quant_config = vllm_config.quant_config
|
||||
|
||||
if config.is_kda_layer(layer_idx):
|
||||
self.self_attn = KimiGatedDeltaNetAttention(
|
||||
config,
|
||||
vllm_config,
|
||||
prefix=f"{prefix}.self_attn",
|
||||
)
|
||||
else:
|
||||
self.self_attn = KimiMLAAttention(
|
||||
layer_idx=layer_idx,
|
||||
hidden_size=self.hidden_size,
|
||||
num_heads=config.num_attention_heads,
|
||||
quant_config=quant_config,
|
||||
cache_config=cache_config,
|
||||
model_config=model_config,
|
||||
prefix=f"{prefix}.self_attn",
|
||||
config=config,
|
||||
qk_nope_head_dim=config.qk_nope_head_dim,
|
||||
qk_rope_head_dim=config.qk_rope_head_dim,
|
||||
v_head_dim=config.v_head_dim,
|
||||
q_lora_rank=config.q_lora_rank,
|
||||
kv_lora_rank=config.kv_lora_rank,
|
||||
use_nope=config.mla_use_nope,
|
||||
)
|
||||
|
||||
if (
|
||||
self.is_moe
|
||||
and config.num_experts is not None
|
||||
and layer_idx >= config.first_k_dense_replace
|
||||
and layer_idx % config.moe_layer_freq == 0
|
||||
):
|
||||
self.block_sparse_moe = KimiMoE(
|
||||
config=config,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.block_sparse_moe",
|
||||
)
|
||||
self.mlp = self.block_sparse_moe
|
||||
else:
|
||||
self.mlp = KimiMLP(
|
||||
hidden_size=self.hidden_size,
|
||||
intermediate_size=config.intermediate_size,
|
||||
hidden_act=config.hidden_act,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.mlp",
|
||||
)
|
||||
self.input_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
self.post_attention_layernorm = RMSNorm(
|
||||
config.hidden_size, eps=config.rms_norm_eps
|
||||
)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
positions: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
residual: torch.Tensor | None,
|
||||
**kwargs,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
# Self Attention
|
||||
if residual is None:
|
||||
residual = hidden_states
|
||||
hidden_states = self.input_layernorm(hidden_states)
|
||||
else:
|
||||
hidden_states, residual = self.input_layernorm(hidden_states, residual)
|
||||
|
||||
attn_output = torch.empty_like(hidden_states)
|
||||
self.self_attn(
|
||||
hidden_states=hidden_states,
|
||||
positions=positions,
|
||||
output=attn_output,
|
||||
)
|
||||
hidden_states = attn_output
|
||||
|
||||
# Fully Connected
|
||||
hidden_states, residual = self.post_attention_layernorm(hidden_states, residual)
|
||||
hidden_states = self.mlp(hidden_states)
|
||||
return hidden_states, residual
|
||||
|
||||
|
||||
@support_torch_compile
|
||||
class KimiLinearModel(nn.Module):
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
super().__init__()
|
||||
|
||||
config = vllm_config.model_config.hf_text_config
|
||||
self.config = config
|
||||
|
||||
self.vocab_size = config.vocab_size
|
||||
|
||||
if get_pp_group().is_first_rank:
|
||||
self.embed_tokens = VocabParallelEmbedding(
|
||||
config.vocab_size,
|
||||
config.hidden_size,
|
||||
prefix=f"{prefix}.embed_tokens",
|
||||
)
|
||||
else:
|
||||
self.embed_tokens = PPMissingLayer()
|
||||
|
||||
def get_layer(prefix: str):
|
||||
return KimiDecoderLayer(
|
||||
config,
|
||||
vllm_config,
|
||||
prefix,
|
||||
)
|
||||
|
||||
self.start_layer, self.end_layer, self.layers = make_layers(
|
||||
config.num_hidden_layers,
|
||||
get_layer,
|
||||
prefix=f"{prefix}.layers",
|
||||
)
|
||||
|
||||
if get_pp_group().is_last_rank:
|
||||
self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
else:
|
||||
self.norm = PPMissingLayer()
|
||||
|
||||
world_size = get_tensor_model_parallel_world_size()
|
||||
assert config.num_attention_heads % world_size == 0, (
|
||||
"num_attention_heads must be divisible by world_size"
|
||||
)
|
||||
|
||||
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
|
||||
return self.embed_tokens(input_ids)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.Tensor | None,
|
||||
positions: torch.Tensor,
|
||||
intermediate_tensors: IntermediateTensors | None,
|
||||
inputs_embeds: torch.Tensor | None = None,
|
||||
**kwargs,
|
||||
) -> torch.Tensor:
|
||||
if get_pp_group().is_first_rank:
|
||||
if inputs_embeds is not None:
|
||||
hidden_states = inputs_embeds
|
||||
else:
|
||||
hidden_states = self.embed_input_ids(input_ids)
|
||||
residual = None
|
||||
else:
|
||||
assert intermediate_tensors is not None
|
||||
hidden_states = intermediate_tensors["hidden_states"]
|
||||
residual = intermediate_tensors["residual"]
|
||||
|
||||
for _, layer in enumerate(self.layers[self.start_layer : self.end_layer]):
|
||||
hidden_states, residual = layer(
|
||||
positions=positions,
|
||||
hidden_states=hidden_states,
|
||||
residual=residual,
|
||||
)
|
||||
|
||||
if not get_pp_group().is_last_rank:
|
||||
return IntermediateTensors(
|
||||
{"hidden_states": hidden_states, "residual": residual}
|
||||
)
|
||||
|
||||
hidden_states, _ = self.norm(hidden_states, residual)
|
||||
return hidden_states
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
stacked_params_mapping = [
|
||||
# (param_name, shard_name, shard_id)
|
||||
(".gate_up_proj", ".gate_proj", 0),
|
||||
(".gate_up_proj", ".up_proj", 1),
|
||||
]
|
||||
if self.config.is_moe:
|
||||
# Params for weights, fp8 weight scales, fp8 activation scales
|
||||
# (param_name, weight_name, expert_id, shard_id)
|
||||
expert_params_mapping = fused_moe_make_expert_params_mapping(
|
||||
self,
|
||||
ckpt_gate_proj_name="w1",
|
||||
ckpt_down_proj_name="w2",
|
||||
ckpt_up_proj_name="w3",
|
||||
num_experts=self.config.num_experts,
|
||||
)
|
||||
else:
|
||||
expert_params_mapping = []
|
||||
params_dict = dict(self.named_parameters())
|
||||
loaded_params: set[str] = set()
|
||||
for args in weights:
|
||||
name, loaded_weight = args[:2]
|
||||
kwargs = args[2] if len(args) > 2 else {}
|
||||
if "rotary_emb.inv_freq" in name:
|
||||
continue
|
||||
|
||||
spec_layer = get_spec_layer_idx_from_weight_name(self.config, name)
|
||||
if spec_layer is not None:
|
||||
continue # skip spec decode layers for main model
|
||||
if "rotary_emb.cos_cached" in name or "rotary_emb.sin_cached" in name:
|
||||
# Models trained using ColossalAI may include these tensors in
|
||||
# the checkpoint. Skip them.
|
||||
continue
|
||||
for param_name, weight_name, shard_id in stacked_params_mapping:
|
||||
if weight_name not in name:
|
||||
continue
|
||||
# We have mlp.experts[0].gate_proj in the checkpoint.
|
||||
# Since we handle the experts below in expert_params_mapping,
|
||||
# we need to skip here BEFORE we update the name, otherwise
|
||||
# name will be updated to mlp.experts[0].gate_up_proj, which
|
||||
# will then be updated below in expert_params_mapping
|
||||
# for mlp.experts[0].gate_gate_up_proj, which breaks load.
|
||||
if ("mlp.experts." in name) and name not in params_dict:
|
||||
continue
|
||||
name = name.replace(weight_name, param_name)
|
||||
# Skip loading extra bias for GPTQ models.
|
||||
if name.endswith(".bias") and name not in params_dict:
|
||||
continue
|
||||
if is_pp_missing_parameter(name, self):
|
||||
continue
|
||||
param = params_dict[name]
|
||||
weight_loader = param.weight_loader
|
||||
weight_loader(param, loaded_weight, shard_id)
|
||||
break
|
||||
else:
|
||||
for idx, (param_name, weight_name, expert_id, shard_id) in enumerate(
|
||||
expert_params_mapping
|
||||
):
|
||||
if weight_name not in name:
|
||||
continue
|
||||
name = name.replace(weight_name, param_name)
|
||||
if is_pp_missing_parameter(name, self):
|
||||
continue
|
||||
param = params_dict[name]
|
||||
weight_loader = param.weight_loader
|
||||
weight_loader(
|
||||
param,
|
||||
loaded_weight,
|
||||
name,
|
||||
expert_id=expert_id,
|
||||
shard_id=shard_id,
|
||||
)
|
||||
break
|
||||
else:
|
||||
# Skip loading extra bias for GPTQ models.
|
||||
if (
|
||||
name.endswith(".bias")
|
||||
and name not in params_dict
|
||||
and not self.config.is_linear_attn
|
||||
): # noqa: E501
|
||||
continue
|
||||
# Remapping the name of FP8 kv-scale.
|
||||
name = maybe_remap_kv_scale_name(name, params_dict)
|
||||
if name is None:
|
||||
continue
|
||||
if is_pp_missing_parameter(name, self):
|
||||
continue
|
||||
|
||||
param = params_dict[name]
|
||||
weight_loader = getattr(
|
||||
param, "weight_loader", default_weight_loader
|
||||
)
|
||||
weight_loader(param, loaded_weight, **kwargs)
|
||||
loaded_params.add(name)
|
||||
return loaded_params
|
||||
|
||||
|
||||
class KimiLinearForCausalLM(
|
||||
nn.Module, HasInnerState, SupportsPP, MixtureOfExperts, IsHybrid
|
||||
):
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
super().__init__()
|
||||
self.model_config = vllm_config.model_config
|
||||
self.vllm_config = vllm_config
|
||||
self.config = self.model_config.hf_config
|
||||
quant_config = vllm_config.quant_config
|
||||
self.quant_config = quant_config
|
||||
self.model = KimiLinearModel(
|
||||
vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model")
|
||||
)
|
||||
if get_pp_group().is_last_rank:
|
||||
self.lm_head = ParallelLMHead(
|
||||
self.config.vocab_size,
|
||||
self.config.hidden_size,
|
||||
quant_config=quant_config,
|
||||
prefix=maybe_prefix(prefix, "lm_head"),
|
||||
)
|
||||
else:
|
||||
self.lm_head = PPMissingLayer()
|
||||
logit_scale = getattr(self.config, "logit_scale", 1.0)
|
||||
self.logits_processor = LogitsProcessor(
|
||||
self.config.vocab_size, scale=logit_scale
|
||||
)
|
||||
|
||||
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
|
||||
return self.model.embed_input_ids(input_ids)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.Tensor | None,
|
||||
positions: torch.Tensor,
|
||||
intermediate_tensors: IntermediateTensors | None = None,
|
||||
inputs_embeds: torch.Tensor | None = None,
|
||||
**kwargs,
|
||||
) -> torch.Tensor | IntermediateTensors:
|
||||
hidden_states = self.model(
|
||||
input_ids, positions, intermediate_tensors, inputs_embeds, **kwargs
|
||||
)
|
||||
return hidden_states
|
||||
|
||||
@classmethod
|
||||
def get_mamba_state_dtype_from_config(
|
||||
cls,
|
||||
vllm_config: "VllmConfig",
|
||||
) -> tuple[torch.dtype, torch.dtype]:
|
||||
return MambaStateDtypeCalculator.kda_state_dtype(
|
||||
vllm_config.model_config.dtype, vllm_config.cache_config.mamba_cache_dtype
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def get_mamba_state_shape_from_config(
|
||||
cls, vllm_config: "VllmConfig"
|
||||
) -> tuple[tuple[int, ...], tuple[int, ...]]:
|
||||
parallel_config = vllm_config.parallel_config
|
||||
hf_config = vllm_config.model_config.hf_config
|
||||
tp_size = parallel_config.tensor_parallel_size
|
||||
num_spec = (
|
||||
vllm_config.speculative_config.num_speculative_tokens
|
||||
if vllm_config.speculative_config
|
||||
else 0
|
||||
)
|
||||
return MambaStateShapeCalculator.kda_state_shape(
|
||||
tp_size,
|
||||
hf_config.linear_attn_config["num_heads"],
|
||||
hf_config.linear_attn_config["head_dim"],
|
||||
conv_kernel_size=hf_config.linear_attn_config["short_conv_kernel_size"],
|
||||
num_spec=num_spec,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def get_mamba_state_copy_func(
|
||||
cls,
|
||||
) -> tuple[MambaStateCopyFunc, MambaStateCopyFunc]:
|
||||
return MambaStateCopyFuncCalculator.kda_state_copy_func()
|
||||
|
||||
def compute_logits(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
) -> torch.Tensor | None:
|
||||
return self.logits_processor(self.lm_head, hidden_states)
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
loader = AutoWeightsLoader(
|
||||
self,
|
||||
skip_prefixes=(["lm_head."] if self.config.tie_word_embeddings else None),
|
||||
)
|
||||
return loader.load_weights(weights)
|
||||
@@ -137,7 +137,10 @@ _TEXT_GENERATION_MODELS = {
|
||||
"IQuestLoopCoderForCausalLM": ("iquest_loopcoder", "IQuestLoopCoderForCausalLM"),
|
||||
"Jais2ForCausalLM": ("jais2", "Jais2ForCausalLM"),
|
||||
"JambaForCausalLM": ("jamba", "JambaForCausalLM"),
|
||||
"KimiLinearForCausalLM": ("kimi_linear", "KimiLinearForCausalLM"),
|
||||
"KimiLinearForCausalLM": (
|
||||
"vllm.models.kimi_k3",
|
||||
"KimiLinearForCausalLM",
|
||||
),
|
||||
"Lfm2ForCausalLM": ("lfm2", "Lfm2ForCausalLM"),
|
||||
"Lfm2MoeForCausalLM": ("lfm2_moe", "Lfm2MoeForCausalLM"),
|
||||
"LagunaForCausalLM": ("laguna", "LagunaForCausalLM"),
|
||||
@@ -460,6 +463,10 @@ _MULTIMODAL_MODELS = {
|
||||
),
|
||||
"KimiVLForConditionalGeneration": ("kimi_vl", "KimiVLForConditionalGeneration"),
|
||||
"KimiK25ForConditionalGeneration": ("kimi_k25", "KimiK25ForConditionalGeneration"),
|
||||
"KimiK3ForConditionalGeneration": (
|
||||
"vllm.models.kimi_k3",
|
||||
"KimiK3ForConditionalGeneration",
|
||||
),
|
||||
"MoonshotKimiaForCausalLM": ("kimi_audio", "KimiAudioForConditionalGeneration"),
|
||||
"MossTranscribeDiarizeForConditionalGeneration": (
|
||||
"moss_transcribe_diarize",
|
||||
@@ -609,6 +616,10 @@ _SPECULATIVE_DECODING_MODELS = {
|
||||
"DFlashDraftModel": ("qwen3_dflash", "DFlashQwen3ForCausalLM"),
|
||||
"DSparkDraftModel": ("vllm.models.deepseek_v4", "DSparkDeepseekV4ForCausalLM"),
|
||||
"Qwen3DSparkModel": ("qwen3_dspark", "Qwen3DSparkForCausalLM"),
|
||||
"K3DSparkModel": (
|
||||
"vllm.models.kimi_k3.nvidia.dspark_mla",
|
||||
"K3DSparkForCausalLM",
|
||||
),
|
||||
"DFlashLagunaForCausalLM": ("laguna_dflash", "DFlashLagunaForCausalLM"),
|
||||
"Gemma4DSparkModel": ("gemma4_dspark", "Gemma4DSparkForCausalLM"),
|
||||
"PEagleDraftModel": ("llama_eagle3", "Eagle3LlamaForCausalLM"),
|
||||
@@ -649,6 +660,7 @@ _SPECULATIVE_DECODING_MODELS = {
|
||||
"Qwen3_5MTP": ("qwen3_5_mtp", "Qwen3_5MTP"),
|
||||
"Qwen3_5MoeMTP": ("qwen3_5_mtp", "Qwen3_5MoeMTP"),
|
||||
"HYV3MTPModel": ("hy_v3_mtp", "HYV3MTP"),
|
||||
"KimiK3MTPModel": ("vllm.models.kimi_k3", "KimiK3MTP"),
|
||||
# Temporarily disabled.
|
||||
# # TODO(woosuk): Re-enable this once the MLP Speculator is supported in V1.
|
||||
# "MLPSpeculatorPreTrainedModel": ("mlp_speculator", "MLPSpeculator"),
|
||||
|
||||
@@ -234,7 +234,7 @@ def _deepgemm_fp8_gemm_nt_warmup(
|
||||
|
||||
device = w.device
|
||||
a1q = torch.empty((max_tokens, k), device=device, dtype=torch.float8_e4m3fn)
|
||||
a1q_scales = torch.empty(
|
||||
a1q_scales = torch.zeros(
|
||||
(max_tokens, k // block_m), device=device, dtype=torch.float32
|
||||
)
|
||||
out = torch.empty((max_tokens, n), device=device, dtype=torch.bfloat16)
|
||||
@@ -336,7 +336,7 @@ def _deepgemm_grouped_fp8_gemm_nt_contiguous_warmup(
|
||||
def _warmup(w: torch.Tensor, w_scale: torch.Tensor):
|
||||
_, n, k = w.size()
|
||||
a1q = torch.empty((MAX_M, k), device=device, dtype=torch.float8_e4m3fn)
|
||||
a1q_scales = torch.empty(
|
||||
a1q_scales = torch.zeros(
|
||||
(MAX_M, k // block_m), device=device, dtype=torch.float32
|
||||
)
|
||||
out = torch.empty((MAX_M, n), device=device, dtype=torch.bfloat16)
|
||||
|
||||
@@ -28,6 +28,9 @@ from vllm.model_executor.warmup.flashinfer_sparse_mla_warmup import (
|
||||
deepseek_v4_sparse_mla_attention_warmup,
|
||||
flashinfer_sparse_mla_decode_autotune_warmup,
|
||||
)
|
||||
from vllm.model_executor.warmup.kimi_k3_triton_warmup import (
|
||||
kimi_k3_triton_warmup,
|
||||
)
|
||||
from vllm.model_executor.warmup.qwen_triton_warmup import qwen_triton_warmup
|
||||
from vllm.model_executor.warmup.sparse_mla_triton_warmup import (
|
||||
sparse_mla_triton_warmup,
|
||||
@@ -123,6 +126,7 @@ def kernel_warmup(worker: "Worker", *, process_local_only: bool = False):
|
||||
|
||||
# Run next so input-prep kernels JIT against pristine runner state.
|
||||
if worker.vllm_config.kernel_config.enable_jit_warmup:
|
||||
kimi_k3_triton_warmup(worker)
|
||||
fa4_cutedsl_warmup(worker)
|
||||
sparse_mla_triton_warmup(worker)
|
||||
|
||||
|
||||
@@ -0,0 +1,182 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Warm up Kimi-K3 Triton kernels."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.models.kimi_k3.nvidia.kda import KimiK3DeltaAttention
|
||||
from vllm.v1.worker.gpu_worker import Worker
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def _get_kda_layer(worker: Worker) -> KimiK3DeltaAttention | None:
|
||||
from vllm.models.kimi_k3.nvidia.kda import KimiK3DeltaAttention
|
||||
|
||||
compilation_config = getattr(
|
||||
worker.model_runner,
|
||||
"compilation_config",
|
||||
None,
|
||||
)
|
||||
static_context = getattr(compilation_config, "static_forward_context", None)
|
||||
if not isinstance(static_context, dict):
|
||||
return None
|
||||
return next(
|
||||
(
|
||||
layer
|
||||
for layer in static_context.values()
|
||||
if isinstance(layer, KimiK3DeltaAttention)
|
||||
),
|
||||
None,
|
||||
)
|
||||
|
||||
|
||||
def _warm_attn_res(worker: Worker) -> None:
|
||||
from vllm.models.kimi_k3.nvidia.ops.attn_res import (
|
||||
attn_res,
|
||||
get_attn_res_triton_warmup_profiles,
|
||||
)
|
||||
|
||||
config = worker.model_config.hf_text_config
|
||||
block_size = getattr(config, "attn_res_block_size", None)
|
||||
if block_size is None:
|
||||
return
|
||||
|
||||
hidden_size = int(config.hidden_size)
|
||||
max_blocks = (int(config.num_hidden_layers) + block_size - 1) // block_size
|
||||
if max_blocks < 2:
|
||||
return
|
||||
|
||||
dtype = worker.model_config.dtype
|
||||
device = torch.device("cuda")
|
||||
eps = float(config.rms_norm_eps)
|
||||
prefix = torch.zeros((1, hidden_size), dtype=dtype, device=device)
|
||||
delta = torch.zeros_like(prefix)
|
||||
blocks = torch.zeros(
|
||||
(1, max_blocks, hidden_size),
|
||||
dtype=dtype,
|
||||
device=device,
|
||||
)
|
||||
norm_weight = torch.zeros(hidden_size, dtype=dtype, device=device)
|
||||
qk_weight = torch.zeros_like(norm_weight)
|
||||
output_norm_weight = torch.zeros_like(norm_weight)
|
||||
|
||||
for (
|
||||
num_blocks,
|
||||
has_delta,
|
||||
block_write_idx,
|
||||
apply_output_norm,
|
||||
) in get_attn_res_triton_warmup_profiles(max_blocks):
|
||||
attn_res(
|
||||
prefix,
|
||||
delta if has_delta else None,
|
||||
blocks,
|
||||
norm_weight,
|
||||
qk_weight,
|
||||
output_norm_weight if apply_output_norm else None,
|
||||
num_blocks=num_blocks,
|
||||
block_write_idx=block_write_idx,
|
||||
eps=eps,
|
||||
output_norm_eps=eps if apply_output_norm else 0.0,
|
||||
)
|
||||
|
||||
|
||||
def _warm_recurrent_kda(
|
||||
layer: KimiK3DeltaAttention,
|
||||
input_dtype: torch.dtype,
|
||||
) -> None:
|
||||
from vllm.models.kimi_k3.nvidia.ops.third_party.kda.fused_recurrent import (
|
||||
fused_recurrent_kda,
|
||||
get_fused_recurrent_kda_fwd_warmup_profiles,
|
||||
)
|
||||
|
||||
num_speculative_tokens = int(layer.num_spec)
|
||||
# fused_recurrent_kda_fwd_kernel is only used by speculative decode.
|
||||
if num_speculative_tokens <= 0:
|
||||
return
|
||||
|
||||
kv_cache = layer.kv_cache
|
||||
if not isinstance(kv_cache, (list, tuple)) or len(kv_cache) < 2:
|
||||
return
|
||||
state = kv_cache[1]
|
||||
if not isinstance(state, torch.Tensor) or not state.numel():
|
||||
return
|
||||
|
||||
logger.info("Warming up Kimi-K3 speculative KDA kernels.")
|
||||
h = int(layer.local_num_heads)
|
||||
d = int(layer.head_dim)
|
||||
tokens_per_sequence = num_speculative_tokens + 1
|
||||
for num_sequences in get_fused_recurrent_kda_fwd_warmup_profiles(h):
|
||||
num_tokens = num_sequences * tokens_per_sequence
|
||||
packed_qkv = torch.empty(
|
||||
(num_tokens, 3 * h * d),
|
||||
dtype=input_dtype,
|
||||
device=state.device,
|
||||
)
|
||||
q, k, v = (
|
||||
tensor.view(1, num_tokens, h, d)
|
||||
for tensor in packed_qkv.split(h * d, dim=-1)
|
||||
)
|
||||
fused_recurrent_kda(
|
||||
q=q,
|
||||
k=k,
|
||||
v=v,
|
||||
raw_g=torch.empty(
|
||||
(1, num_tokens, h, d),
|
||||
dtype=input_dtype,
|
||||
device=state.device,
|
||||
),
|
||||
raw_beta=torch.empty(
|
||||
(1, num_tokens, h),
|
||||
dtype=input_dtype,
|
||||
device=state.device,
|
||||
),
|
||||
A_log=layer.A_log,
|
||||
dt_bias=layer.dt_bias,
|
||||
lower_bound=layer.gate_lower_bound,
|
||||
initial_state=state[:1],
|
||||
cu_seqlens=torch.arange(
|
||||
0,
|
||||
num_tokens + 1,
|
||||
tokens_per_sequence,
|
||||
dtype=torch.int32,
|
||||
device=state.device,
|
||||
),
|
||||
ssm_state_indices=torch.zeros(
|
||||
(num_sequences, tokens_per_sequence),
|
||||
dtype=torch.int32,
|
||||
device=state.device,
|
||||
),
|
||||
num_accepted_tokens=torch.ones(
|
||||
num_sequences,
|
||||
dtype=torch.int32,
|
||||
device=state.device,
|
||||
),
|
||||
out=torch.empty(
|
||||
(1, num_tokens, h, d),
|
||||
dtype=input_dtype,
|
||||
device=state.device,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@torch.inference_mode()
|
||||
def kimi_k3_triton_warmup(worker: Worker) -> None:
|
||||
"""Warm Kimi-K3 Triton kernels reachable by this server."""
|
||||
if not current_platform.is_cuda():
|
||||
return
|
||||
|
||||
layer = _get_kda_layer(worker)
|
||||
if layer is None:
|
||||
return
|
||||
|
||||
_warm_attn_res(worker)
|
||||
_warm_recurrent_kda(layer, worker.model_config.dtype)
|
||||
Reference in New Issue
Block a user