From 07ef21bc69842665cf5056371ea15a6d014f3b05 Mon Sep 17 00:00:00 2001 From: kiroxu <148877251+BabyDrangoner@users.noreply.github.com> Date: Tue, 25 Aug 2026 09:28:49 +0800 Subject: [PATCH] [Kernel][Perf] Enable fused QK-norm + partial MRoPE + gate for Qwen3.6 (#52676) Signed-off-by: kiroxu <148877251+BabyDrangoner@users.noreply.github.com> Co-authored-by: OpenAI Codex --- tests/kernels/test_fused_qk_norm_rope_gate.py | 193 +++++++----------- .../layers/fused_qk_norm_rope.py | 97 ++++++++- vllm/model_executor/models/qwen3_next.py | 38 +++- 3 files changed, 191 insertions(+), 137 deletions(-) diff --git a/tests/kernels/test_fused_qk_norm_rope_gate.py b/tests/kernels/test_fused_qk_norm_rope_gate.py index 09ec90d488..b743f1ace7 100644 --- a/tests/kernels/test_fused_qk_norm_rope_gate.py +++ b/tests/kernels/test_fused_qk_norm_rope_gate.py @@ -5,155 +5,114 @@ import pytest import torch from vllm.model_executor.layers.fused_qk_norm_rope import fused_qk_rmsnorm_rope_gate -from vllm.model_executor.layers.rotary_embedding import RotaryEmbedding +from vllm.model_executor.layers.layernorm import GemmaRMSNorm +from vllm.model_executor.layers.rotary_embedding import ( + MRotaryEmbedding, + RotaryEmbedding, +) from vllm.platforms import current_platform from vllm.utils.torch_utils import set_random_seed -# Qwen/Qwen3.6-27B config (huggingface.co/Qwen/Qwen3.6-27B), TP=1 shapes. -NUM_Q_HEADS = 24 -NUM_KV_HEADS = 4 +# Qwen3.6 TP=1 attention geometry. HEAD_DIM = 256 -PARTIAL_ROTARY_FACTOR = 0.25 -ROTARY_DIM = int(HEAD_DIM * PARTIAL_ROTARY_FACTOR) # 64 +ROTARY_DIM = 64 RMS_NORM_EPS = 1e-6 MAX_POSITION_EMBEDDINGS = 262144 ROPE_THETA = 10000000.0 - -DTYPES = [torch.bfloat16] -SEEDS = [13] -NUM_TOKENS = [1, 4, 37] - - -def _ref_qk_rmsnorm_rope_gate( - q_gate: torch.Tensor, - k: torch.Tensor, - q_gamma: torch.Tensor, - k_gamma: torch.Tensor, - cos_sin_cache: torch.Tensor, - positions: torch.Tensor, - eps: float, - num_q_heads: int, - num_kv_heads: int, - head_dim: int, - rotary_dim: int, -) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: - """PyTorch reference: split + RMSNorm + partial NeoX RoPE + gate extraction. - - Matches ``fused_qk_rmsnorm_rope_gate``'s contract: ``q_gamma`` / ``k_gamma`` - are the already-adjusted effective gammas (for GemmaRMSNorm the caller - has done ``weight + 1`` before passing them in). - """ - n_tokens = q_gate.shape[0] - half = rotary_dim // 2 - - # Per head the q projection is laid out as [q | gate]. - q_gate = q_gate.view(n_tokens, num_q_heads, 2 * head_dim) - q = q_gate[..., :head_dim] - gate = q_gate[..., head_dim:].reshape(n_tokens, num_q_heads * head_dim) - k = k.view(n_tokens, num_kv_heads, head_dim) - - def rms_norm(x: torch.Tensor, gamma: torch.Tensor) -> torch.Tensor: - orig_dtype = x.dtype - x = x.float() - var = x.pow(2).mean(dim=-1, keepdim=True) - x = x * torch.rsqrt(var + eps) - return (x * gamma.float()).to(orig_dtype) - - q = rms_norm(q, q_gamma) - k = rms_norm(k, k_gamma) - - # Partial NeoX RoPE on the first ``rotary_dim`` elements of each head; - # cos_sin_cache row is packed as [cos(half) | sin(half)]. - pos = positions.view(-1) - cos = cos_sin_cache[pos, :half].float()[:, None, :] - sin = cos_sin_cache[pos, half:rotary_dim].float()[:, None, :] - - def rope(x: torch.Tensor) -> torch.Tensor: - x_rot, x_pass = x[..., :rotary_dim], x[..., rotary_dim:] - x1 = x_rot[..., :half].float() - x2 = x_rot[..., half:].float() - o1 = x1 * cos - x2 * sin - o2 = x2 * cos + x1 * sin - rotated = torch.cat([o1, o2], dim=-1).to(x.dtype) - return torch.cat([rotated, x_pass], dim=-1) - - q = rope(q).reshape(n_tokens, num_q_heads * head_dim) - k = rope(k).reshape(n_tokens, num_kv_heads * head_dim) - return q, k, gate +DTYPE = torch.bfloat16 +SEED = 13 +MROPE_SECTION = (11, 11, 10) +ROPE_CASES = [ + pytest.param(24, 4, None, id="rope"), + pytest.param(16, 2, MROPE_SECTION, id="interleaved-mrope"), +] @pytest.mark.skipif( not current_platform.is_cuda_alike(), reason="fused_qk_rmsnorm_rope_gate Triton kernel requires CUDA/ROCm", ) -@pytest.mark.parametrize("dtype", DTYPES) -@pytest.mark.parametrize("seed", SEEDS) -@pytest.mark.parametrize("num_tokens", NUM_TOKENS) +@pytest.mark.parametrize("num_q_heads,num_kv_heads,mrope_section", ROPE_CASES) +@pytest.mark.parametrize("num_tokens", [1, 4, 37]) @torch.inference_mode() def test_fused_qk_norm_rope_gate_matches_reference( default_vllm_config, - dtype: torch.dtype, - seed: int, num_tokens: int, -): + num_q_heads: int, + num_kv_heads: int, + mrope_section: tuple[int, int, int] | None, +) -> None: device = torch.device("cuda", torch.accelerator.current_device_index()) torch.set_default_device(device) - set_random_seed(seed) + set_random_seed(SEED) q_gate = torch.randn( - num_tokens, NUM_Q_HEADS * 2 * HEAD_DIM, dtype=dtype, device=device - ) - k = torch.randn(num_tokens, NUM_KV_HEADS * HEAD_DIM, dtype=dtype, device=device) - # GemmaRMSNorm-style: the kernel takes the effective gamma (weight + 1). - q_gamma = ( - torch.empty(HEAD_DIM, dtype=dtype, device=device).normal_(mean=0.0, std=0.1) - + 1.0 - ) - k_gamma = ( - torch.empty(HEAD_DIM, dtype=dtype, device=device).normal_(mean=0.0, std=0.1) - + 1.0 + num_tokens, num_q_heads * 2 * HEAD_DIM, dtype=DTYPE, device=device ) + k = torch.randn(num_tokens, num_kv_heads * HEAD_DIM, dtype=DTYPE, device=device) - # fused_qk_rmsnorm_rope_gate only handles NeoX-style RoPE. - rope = RotaryEmbedding( - head_size=HEAD_DIM, - rotary_dim=ROTARY_DIM, - max_position_embeddings=MAX_POSITION_EMBEDDINGS, - base=ROPE_THETA, - is_neox_style=True, - dtype=dtype, - ).to(device) - positions = torch.arange(num_tokens, dtype=torch.long, device=device) + q_norm = GemmaRMSNorm(HEAD_DIM, eps=RMS_NORM_EPS).to(device, dtype=DTYPE) + k_norm = GemmaRMSNorm(HEAD_DIM, eps=RMS_NORM_EPS).to(device, dtype=DTYPE) + q_norm.weight.normal_(std=0.1) + k_norm.weight.normal_(std=0.1) + + q_gate_heads = q_gate.view(num_tokens, num_q_heads, 2 * HEAD_DIM) + q = q_gate_heads[..., :HEAD_DIM] + gate_ref = q_gate_heads[..., HEAD_DIM:].reshape(num_tokens, num_q_heads * HEAD_DIM) + q_ref = q_norm.forward_native(q) + k_ref = k_norm.forward_native(k.view(num_tokens, num_kv_heads, HEAD_DIM)) + assert isinstance(q_ref, torch.Tensor) + assert isinstance(k_ref, torch.Tensor) + q_ref = q_ref.reshape(num_tokens, num_q_heads * HEAD_DIM) + k_ref = k_ref.reshape(num_tokens, num_kv_heads * HEAD_DIM) + + if mrope_section is None: + rope = RotaryEmbedding( + HEAD_DIM, + ROTARY_DIM, + MAX_POSITION_EMBEDDINGS, + ROPE_THETA, + True, + DTYPE, + ).to(device) + positions = torch.arange(num_tokens, dtype=torch.long, device=device) + else: + rope = MRotaryEmbedding( + HEAD_DIM, + ROTARY_DIM, + MAX_POSITION_EMBEDDINGS, + ROPE_THETA, + True, + DTYPE, + mrope_section=list(mrope_section), + mrope_interleaved=True, + ).to(device) + positions = torch.arange(3 * num_tokens, dtype=torch.long, device=device).view( + 3, num_tokens + ) + assert torch.unique(positions[:, 0]).numel() == 3 + + q_ref, k_ref = rope.forward_native(positions, q_ref, k_ref) + assert k_ref is not None - q_ref, k_ref, gate_ref = _ref_qk_rmsnorm_rope_gate( - q_gate, - k, - q_gamma, - k_gamma, - rope.cos_sin_cache, - positions, - RMS_NORM_EPS, - NUM_Q_HEADS, - NUM_KV_HEADS, - HEAD_DIM, - ROTARY_DIM, - ) q_out, k_out, gate_out = fused_qk_rmsnorm_rope_gate( q_gate, k, - q_gamma, - k_gamma, + q_norm.weight, + k_norm.weight, rope.cos_sin_cache, positions, RMS_NORM_EPS, - NUM_Q_HEADS, - NUM_KV_HEADS, + num_q_heads, + num_kv_heads, HEAD_DIM, ROTARY_DIM, + mrope_section=mrope_section, + norm_beta=1.0, ) - atol, rtol = 2e-3, 5e-3 - torch.testing.assert_close(q_out, q_ref, atol=atol, rtol=rtol) - torch.testing.assert_close(k_out, k_ref, atol=atol, rtol=rtol) - # gate is a verbatim copy of the source slice — must match bit-exactly. + # The built-in reference performs RoPE in BF16, while the fused kernel + # promotes the BF16-normalized values to FP32 for RoPE before storing BF16. + torch.testing.assert_close(q_out, q_ref, atol=1e-2, rtol=1e-2) + torch.testing.assert_close(k_out, k_ref, atol=1e-2, rtol=1e-2) torch.testing.assert_close(gate_out, gate_ref, atol=0, rtol=0) diff --git a/vllm/model_executor/layers/fused_qk_norm_rope.py b/vllm/model_executor/layers/fused_qk_norm_rope.py index 21549dd783..d3c645488a 100644 --- a/vllm/model_executor/layers/fused_qk_norm_rope.py +++ b/vllm/model_executor/layers/fused_qk_norm_rope.py @@ -30,16 +30,22 @@ def _fused_qk_rmsnorm_rope_gate_kernel( k_out_stride_t, gate_out_stride_t, cache_stride_p, + positions_stride_m, + positions_stride_t, num_q_heads: tl.constexpr, num_kv_heads: tl.constexpr, head_dim: tl.constexpr, rotary_dim: tl.constexpr, half_rotary: tl.constexpr, eps: tl.constexpr, + norm_beta: tl.constexpr, INPUT_DTYPE: tl.constexpr, HEAD_BLOCK: tl.constexpr, ROT_HALF_BLOCK: tl.constexpr, HAS_PASS: tl.constexpr, + HAS_MROPE: tl.constexpr, + MROPE_SECTION_H: tl.constexpr, + MROPE_SECTION_W: tl.constexpr, ): token = tl.program_id(0) head = tl.program_id(1) @@ -61,7 +67,7 @@ def _fused_qk_rmsnorm_rope_gate_kernel( x = tl.load(in_base + head_offs, mask=head_mask, other=0.0).to(tl.float32) var = tl.sum(x * x, axis=0) / head_dim inv_rms = tl.rsqrt(var + eps) - w = tl.load(w_ptr + head_offs, mask=head_mask, other=0.0).to(tl.float32) + w = tl.load(w_ptr + head_offs, mask=head_mask, other=0.0).to(tl.float32) + norm_beta # Round-trip through INPUT_DTYPE so the RoPE input matches the bf16-storage # behavior of the unfused (qk_rmsnorm -> memory -> apply_rope) reference path. x_norm = (x * inv_rms * w).to(INPUT_DTYPE).to(tl.float32) @@ -82,15 +88,30 @@ def _fused_qk_rmsnorm_rope_gate_kernel( x_rot2 = tl.load(in_base + half_rotary + rot_offs, mask=rot_mask, other=0.0).to( tl.float32 ) - w_rot1 = tl.load(w_ptr + rot_offs, mask=rot_mask, other=0.0).to(tl.float32) - w_rot2 = tl.load(w_ptr + half_rotary + rot_offs, mask=rot_mask, other=0.0).to( - tl.float32 + w_rot1 = ( + tl.load(w_ptr + rot_offs, mask=rot_mask, other=0.0).to(tl.float32) + norm_beta + ) + w_rot2 = ( + tl.load(w_ptr + half_rotary + rot_offs, mask=rot_mask, other=0.0).to(tl.float32) + + norm_beta ) x_rot1 = (x_rot1 * inv_rms * w_rot1).to(INPUT_DTYPE).to(tl.float32) x_rot2 = (x_rot2 * inv_rms * w_rot2).to(INPUT_DTYPE).to(tl.float32) - # Always use int64 for position to avoid overflow in address computation. - pos = tl.load(positions_ptr + token).to(tl.int64) + # Always use int64 for positions to avoid overflow in address computation. + pos_t = tl.load(positions_ptr + token * positions_stride_t).to(tl.int64) + if HAS_MROPE: + pos_h = tl.load( + positions_ptr + positions_stride_m + token * positions_stride_t + ).to(tl.int64) + pos_w = tl.load( + positions_ptr + 2 * positions_stride_m + token * positions_stride_t + ).to(tl.int64) + is_h = (rot_offs % 3 == 1) & (rot_offs < 3 * MROPE_SECTION_H) + is_w = (rot_offs % 3 == 2) & (rot_offs < 3 * MROPE_SECTION_W) + pos = tl.where(is_h, pos_h, tl.where(is_w, pos_w, pos_t)) + else: + pos = pos_t cache_offset = pos * cache_stride_p cos = tl.load( cos_sin_cache_ptr + cache_offset + rot_offs, mask=rot_mask, other=0.0 @@ -126,21 +147,25 @@ def fused_qk_rmsnorm_rope_gate( num_kv_heads: int, head_dim: int, rotary_dim: int, + mrope_section: list[int] | tuple[int, int, int] | None = None, + norm_beta: float = 0.0, ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: - """Fused split + QK-RMSNorm + (partial) RoPE + gate copy for Qwen3.5 attn. + """Fused split + QK-RMSNorm + (partial) RoPE + gate copy for Qwen attn. Args: q_gate: (n_tokens, num_q_heads * 2 * head_dim) -- per head: [q|gate] k: (n_tokens, num_kv_heads * head_dim) - q_weight: (head_dim,) GemmaRMSNorm effective weight (already +1) - k_weight: (head_dim,) GemmaRMSNorm effective weight (already +1) + q_weight: (head_dim,) RMSNorm weight + k_weight: (head_dim,) RMSNorm weight cos_sin_cache: (max_pos, rotary_dim) packed [cos|sin] - positions: (n_tokens,) int32 or int64 + positions: (n_tokens,) or (3, n_tokens) int32 or int64 eps: RMSNorm epsilon num_q_heads: number of Q heads (after TP split) num_kv_heads: number of KV heads (after TP split) head_dim: per-head dimension rotary_dim: rotary dimension; must be even and <= head_dim + mrope_section: interleaved T/H/W frequency counts for 2D positions + norm_beta: scalar added to the RMSNorm weight Returns: (q_out, k_out, gate_out) -- all contiguous (n_tokens, heads * head_dim). @@ -151,6 +176,52 @@ def fused_qk_rmsnorm_rope_gate( f"rotary_dim must be a positive even integer <= head_dim, " f"got rotary_dim={rotary_dim}, head_dim={head_dim}" ) + if q_gate.dtype not in (torch.float16, torch.bfloat16) or k.dtype != q_gate.dtype: + raise ValueError( + "q_gate and k must have the same FP16 or BF16 dtype, " + f"got {q_gate.dtype} and {k.dtype}" + ) + for name, tensor in ( + ("q_gate", q_gate), + ("k", k), + ("q_weight", q_weight), + ("k_weight", k_weight), + ("cos_sin_cache", cos_sin_cache), + ): + if tensor.stride(-1) != 1: + raise ValueError(f"{name} must be contiguous in its last dimension") + + if positions.ndim not in (1, 2): + raise ValueError(f"positions must be 1D or 2D, got shape={positions.shape}") + if positions.shape[-1] != q_gate.shape[0]: + raise ValueError( + "positions token dimension must match q_gate, " + f"got {positions.shape[-1]} and {q_gate.shape[0]}" + ) + + has_mrope = positions.ndim == 2 + if has_mrope: + if positions.shape[0] != 3: + raise ValueError( + f"MRoPE positions must have shape (3, n_tokens), got {positions.shape}" + ) + if mrope_section is None or len(mrope_section) != 3: + raise ValueError("mrope_section must contain the T/H/W frequency counts") + if sum(mrope_section) != rotary_dim // 2: + raise ValueError( + "mrope_section must sum to rotary_dim // 2, " + f"got {mrope_section} and rotary_dim={rotary_dim}" + ) + mrope_section_h = mrope_section[1] + mrope_section_w = mrope_section[2] + positions_stride_m, positions_stride_t = positions.stride() + else: + if mrope_section is not None: + raise ValueError("mrope_section requires 2D MRoPE positions") + mrope_section_h = 0 + mrope_section_w = 0 + positions_stride_m = 0 + positions_stride_t = positions.stride(0) n_tokens = q_gate.shape[0] q_out = torch.empty( @@ -185,16 +256,22 @@ def fused_qk_rmsnorm_rope_gate( k_out.stride(0), gate_out.stride(0), cos_sin_cache.stride(0), + positions_stride_m, + positions_stride_t, num_q_heads, num_kv_heads, head_dim, rotary_dim, half_rotary, eps, + norm_beta=norm_beta, INPUT_DTYPE=tl.bfloat16 if q_gate.dtype == torch.bfloat16 else tl.float16, HEAD_BLOCK=head_block, ROT_HALF_BLOCK=rot_half_block, HAS_PASS=rotary_dim < head_dim, + HAS_MROPE=has_mrope, + MROPE_SECTION_H=mrope_section_h, + MROPE_SECTION_W=mrope_section_w, num_warps=num_warps, num_stages=2, ) diff --git a/vllm/model_executor/models/qwen3_next.py b/vllm/model_executor/models/qwen3_next.py index 51fd036f4e..3dc38f3a2d 100644 --- a/vllm/model_executor/models/qwen3_next.py +++ b/vllm/model_executor/models/qwen3_next.py @@ -43,7 +43,7 @@ from vllm.model_executor.layers.mamba.mamba_utils import ( MambaStateShapeCalculator, ) from vllm.model_executor.layers.quantization import QuantizationConfig -from vllm.model_executor.layers.rotary_embedding import get_rope +from vllm.model_executor.layers.rotary_embedding import MRotaryEmbedding, get_rope from vllm.model_executor.layers.vocab_parallel_embedding import ( ParallelLMHead, VocabParallelEmbedding, @@ -300,14 +300,26 @@ class Qwen3NextAttention(nn.Module): self.k_norm = Qwen3NextRMSNorm(self.head_dim, eps=config.rms_norm_eps) # Fuse the gated split + QK-RMSNorm + (partial) NeoX RoPE + gate copy. - # TODO: support MRoPE mm_config = model_config.multimodal_config if model_config else None text_only = mm_config is None or mm_config.language_model_only + mrope_section = getattr(self.rotary_emb, "mrope_section", None) + supports_mrope = bool( + type(self.rotary_emb) is MRotaryEmbedding + and mrope_section + and len(mrope_section) == 3 + and sum(mrope_section) == self.rotary_emb.rotary_dim // 2 + and getattr(self.rotary_emb, "mrope_interleaved", False) + ) + supports_dtype = getattr(self.rotary_emb, "dtype", None) in ( + torch.float16, + torch.bfloat16, + ) self.use_fused_qk_norm_rope_gate = ( self.attn_output_gate and getattr(self.rotary_emb, "is_neox_style", False) and current_platform.is_cuda() - and text_only + and supports_dtype + and (text_only or supports_mrope) ) def _project_qkv_gate( @@ -325,22 +337,28 @@ class Qwen3NextAttention(nn.Module): q_gate, k, v = qkv.split( [self.q_size * 2, self.kv_size, self.kv_size], dim=-1 ) - # mRoPE passes positions as (3, n_tokens) for T/H/W. Fusion is only - # enabled text-only, where the three rows are identical, so taking - # the T row is exact. (1D positions pass through.) - pos = positions[0] if positions.ndim == 2 else positions + if positions.ndim == 2 and not getattr( + self.rotary_emb, "mrope_section", None + ): + positions = positions[0] q, k, gate = fused_qk_rmsnorm_rope_gate( q_gate, k, - self.q_norm.weight.float() + 1.0, - self.k_norm.weight.float() + 1.0, + self.q_norm.weight, + self.k_norm.weight, self.rotary_emb.cos_sin_cache, - pos, + positions, self.q_norm.variance_epsilon, self.num_heads, self.num_kv_heads, self.head_dim, self.rotary_emb.rotary_dim, + norm_beta=1.0, + mrope_section=( + getattr(self.rotary_emb, "mrope_section", None) + if positions.ndim == 2 + else None + ), ) return q, k, v, gate