[Perf] fuse qk rmsnorm rope gate for qwen3.5 (#44176)
Signed-off-by: zjy0516 <riverclouds.zhu@qq.com>
This commit is contained in:
@@ -28,6 +28,7 @@ from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoE,
|
||||
fused_moe_make_expert_params_mapping,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_qk_norm_rope import fused_qk_rmsnorm_rope_gate
|
||||
from vllm.model_executor.layers.layernorm import (
|
||||
GemmaRMSNorm as Qwen3NextRMSNorm,
|
||||
)
|
||||
@@ -58,6 +59,7 @@ from vllm.model_executor.model_loader.weight_utils import (
|
||||
)
|
||||
from vllm.model_executor.models.qwen2_moe import Qwen2MoeMLP as Qwen3NextMLP
|
||||
from vllm.model_executor.models.utils import sequence_parallel_chunk
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.transformers_utils.configs.qwen3_next import Qwen3NextConfig
|
||||
|
||||
@@ -283,13 +285,50 @@ class Qwen3NextAttention(nn.Module):
|
||||
self.q_norm = Qwen3NextRMSNorm(self.head_dim, eps=config.rms_norm_eps)
|
||||
self.k_norm = Qwen3NextRMSNorm(self.head_dim, eps=config.rms_norm_eps)
|
||||
|
||||
def forward(
|
||||
# Fuse the gated split + QK-RMSNorm + (partial) NeoX RoPE + gate copy.
|
||||
# TODO: support MRoPE
|
||||
mm_config = model_config.multimodal_config if model_config else None
|
||||
text_only = mm_config is None or mm_config.language_model_only
|
||||
self.use_fused_qk_norm_rope_gate = (
|
||||
self.attn_output_gate
|
||||
and getattr(self.rotary_emb, "is_neox_style", False)
|
||||
and current_platform.is_cuda()
|
||||
and text_only
|
||||
)
|
||||
|
||||
def _project_qkv_gate(
|
||||
self,
|
||||
qkv: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
output: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
):
|
||||
qkv, _ = self.qkv_proj(hidden_states)
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor | None]:
|
||||
"""Return post-norm, post-RoPE (q, k, v) and the pre-sigmoid gate.
|
||||
|
||||
Dispatches between the fused Triton kernel and the eager
|
||||
split + QK-RMSNorm + RoPE path. ``gate`` is ``None`` when output
|
||||
gating is disabled.
|
||||
"""
|
||||
if self.use_fused_qk_norm_rope_gate:
|
||||
q_gate, k, v = qkv.split(
|
||||
[self.q_size * 2, self.kv_size, self.kv_size], dim=-1
|
||||
)
|
||||
# mRoPE passes positions as (3, n_tokens) for T/H/W. Fusion is only
|
||||
# enabled text-only, where the three rows are identical, so taking
|
||||
# the T row is exact. (1D positions pass through.)
|
||||
pos = positions[0] if positions.ndim == 2 else positions
|
||||
q, k, gate = fused_qk_rmsnorm_rope_gate(
|
||||
q_gate,
|
||||
k,
|
||||
self.q_norm.weight.float() + 1.0,
|
||||
self.k_norm.weight.float() + 1.0,
|
||||
self.rotary_emb.cos_sin_cache,
|
||||
pos,
|
||||
self.q_norm.variance_epsilon,
|
||||
self.num_heads,
|
||||
self.num_kv_heads,
|
||||
self.head_dim,
|
||||
self.rotary_emb.rotary_dim,
|
||||
)
|
||||
return q, k, v, gate
|
||||
|
||||
if self.attn_output_gate:
|
||||
q_gate, k, v = qkv.split(
|
||||
@@ -302,6 +341,7 @@ class Qwen3NextAttention(nn.Module):
|
||||
gate = gate.reshape(*orig_shape, -1)
|
||||
else:
|
||||
q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
|
||||
gate = None
|
||||
|
||||
q = self.q_norm(q.view(-1, self.num_heads, self.head_dim)).view(
|
||||
-1, self.num_heads * self.head_dim
|
||||
@@ -309,15 +349,20 @@ class Qwen3NextAttention(nn.Module):
|
||||
k = self.k_norm(k.view(-1, self.num_kv_heads, self.head_dim)).view(
|
||||
-1, self.num_kv_heads * self.head_dim
|
||||
)
|
||||
|
||||
q, k = self.rotary_emb(positions, q, k)
|
||||
return q, k, v, gate
|
||||
|
||||
def forward(
|
||||
self,
|
||||
positions: torch.Tensor,
|
||||
output: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
):
|
||||
qkv, _ = self.qkv_proj(hidden_states)
|
||||
q, k, v, gate = self._project_qkv_gate(qkv, positions)
|
||||
attn_output = self.attn(q, k, v)
|
||||
|
||||
if self.attn_output_gate:
|
||||
gate = torch.sigmoid(gate)
|
||||
attn_output = attn_output * gate
|
||||
|
||||
if gate is not None:
|
||||
attn_output = attn_output * torch.sigmoid(gate)
|
||||
output[:], _ = self.o_proj(attn_output)
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user