[Kernel][Perf] Enable fused QK-norm + partial MRoPE + gate for Qwen3.6 (#52676)

Signed-off-by: kiroxu <148877251+BabyDrangoner@users.noreply.github.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
kiroxu
2026-08-24 18:28:49 -07:00
committed by GitHub
co-authored by OpenAI Codex
parent 9bf0d4717e
commit 07ef21bc69
3 changed files with 191 additions and 137 deletions
+76 -117
View File
@@ -5,155 +5,114 @@ import pytest
import torch
from vllm.model_executor.layers.fused_qk_norm_rope import fused_qk_rmsnorm_rope_gate
from vllm.model_executor.layers.rotary_embedding import RotaryEmbedding
from vllm.model_executor.layers.layernorm import GemmaRMSNorm
from vllm.model_executor.layers.rotary_embedding import (
MRotaryEmbedding,
RotaryEmbedding,
)
from vllm.platforms import current_platform
from vllm.utils.torch_utils import set_random_seed
# Qwen/Qwen3.6-27B config (huggingface.co/Qwen/Qwen3.6-27B), TP=1 shapes.
NUM_Q_HEADS = 24
NUM_KV_HEADS = 4
# Qwen3.6 TP=1 attention geometry.
HEAD_DIM = 256
PARTIAL_ROTARY_FACTOR = 0.25
ROTARY_DIM = int(HEAD_DIM * PARTIAL_ROTARY_FACTOR) # 64
ROTARY_DIM = 64
RMS_NORM_EPS = 1e-6
MAX_POSITION_EMBEDDINGS = 262144
ROPE_THETA = 10000000.0
DTYPES = [torch.bfloat16]
SEEDS = [13]
NUM_TOKENS = [1, 4, 37]
def _ref_qk_rmsnorm_rope_gate(
q_gate: torch.Tensor,
k: torch.Tensor,
q_gamma: torch.Tensor,
k_gamma: torch.Tensor,
cos_sin_cache: torch.Tensor,
positions: torch.Tensor,
eps: float,
num_q_heads: int,
num_kv_heads: int,
head_dim: int,
rotary_dim: int,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
"""PyTorch reference: split + RMSNorm + partial NeoX RoPE + gate extraction.
Matches ``fused_qk_rmsnorm_rope_gate``'s contract: ``q_gamma`` / ``k_gamma``
are the already-adjusted effective gammas (for GemmaRMSNorm the caller
has done ``weight + 1`` before passing them in).
"""
n_tokens = q_gate.shape[0]
half = rotary_dim // 2
# Per head the q projection is laid out as [q | gate].
q_gate = q_gate.view(n_tokens, num_q_heads, 2 * head_dim)
q = q_gate[..., :head_dim]
gate = q_gate[..., head_dim:].reshape(n_tokens, num_q_heads * head_dim)
k = k.view(n_tokens, num_kv_heads, head_dim)
def rms_norm(x: torch.Tensor, gamma: torch.Tensor) -> torch.Tensor:
orig_dtype = x.dtype
x = x.float()
var = x.pow(2).mean(dim=-1, keepdim=True)
x = x * torch.rsqrt(var + eps)
return (x * gamma.float()).to(orig_dtype)
q = rms_norm(q, q_gamma)
k = rms_norm(k, k_gamma)
# Partial NeoX RoPE on the first ``rotary_dim`` elements of each head;
# cos_sin_cache row is packed as [cos(half) | sin(half)].
pos = positions.view(-1)
cos = cos_sin_cache[pos, :half].float()[:, None, :]
sin = cos_sin_cache[pos, half:rotary_dim].float()[:, None, :]
def rope(x: torch.Tensor) -> torch.Tensor:
x_rot, x_pass = x[..., :rotary_dim], x[..., rotary_dim:]
x1 = x_rot[..., :half].float()
x2 = x_rot[..., half:].float()
o1 = x1 * cos - x2 * sin
o2 = x2 * cos + x1 * sin
rotated = torch.cat([o1, o2], dim=-1).to(x.dtype)
return torch.cat([rotated, x_pass], dim=-1)
q = rope(q).reshape(n_tokens, num_q_heads * head_dim)
k = rope(k).reshape(n_tokens, num_kv_heads * head_dim)
return q, k, gate
DTYPE = torch.bfloat16
SEED = 13
MROPE_SECTION = (11, 11, 10)
ROPE_CASES = [
pytest.param(24, 4, None, id="rope"),
pytest.param(16, 2, MROPE_SECTION, id="interleaved-mrope"),
]
@pytest.mark.skipif(
not current_platform.is_cuda_alike(),
reason="fused_qk_rmsnorm_rope_gate Triton kernel requires CUDA/ROCm",
)
@pytest.mark.parametrize("dtype", DTYPES)
@pytest.mark.parametrize("seed", SEEDS)
@pytest.mark.parametrize("num_tokens", NUM_TOKENS)
@pytest.mark.parametrize("num_q_heads,num_kv_heads,mrope_section", ROPE_CASES)
@pytest.mark.parametrize("num_tokens", [1, 4, 37])
@torch.inference_mode()
def test_fused_qk_norm_rope_gate_matches_reference(
default_vllm_config,
dtype: torch.dtype,
seed: int,
num_tokens: int,
):
num_q_heads: int,
num_kv_heads: int,
mrope_section: tuple[int, int, int] | None,
) -> None:
device = torch.device("cuda", torch.accelerator.current_device_index())
torch.set_default_device(device)
set_random_seed(seed)
set_random_seed(SEED)
q_gate = torch.randn(
num_tokens, NUM_Q_HEADS * 2 * HEAD_DIM, dtype=dtype, device=device
)
k = torch.randn(num_tokens, NUM_KV_HEADS * HEAD_DIM, dtype=dtype, device=device)
# GemmaRMSNorm-style: the kernel takes the effective gamma (weight + 1).
q_gamma = (
torch.empty(HEAD_DIM, dtype=dtype, device=device).normal_(mean=0.0, std=0.1)
+ 1.0
)
k_gamma = (
torch.empty(HEAD_DIM, dtype=dtype, device=device).normal_(mean=0.0, std=0.1)
+ 1.0
num_tokens, num_q_heads * 2 * HEAD_DIM, dtype=DTYPE, device=device
)
k = torch.randn(num_tokens, num_kv_heads * HEAD_DIM, dtype=DTYPE, device=device)
# fused_qk_rmsnorm_rope_gate only handles NeoX-style RoPE.
rope = RotaryEmbedding(
head_size=HEAD_DIM,
rotary_dim=ROTARY_DIM,
max_position_embeddings=MAX_POSITION_EMBEDDINGS,
base=ROPE_THETA,
is_neox_style=True,
dtype=dtype,
).to(device)
positions = torch.arange(num_tokens, dtype=torch.long, device=device)
q_norm = GemmaRMSNorm(HEAD_DIM, eps=RMS_NORM_EPS).to(device, dtype=DTYPE)
k_norm = GemmaRMSNorm(HEAD_DIM, eps=RMS_NORM_EPS).to(device, dtype=DTYPE)
q_norm.weight.normal_(std=0.1)
k_norm.weight.normal_(std=0.1)
q_gate_heads = q_gate.view(num_tokens, num_q_heads, 2 * HEAD_DIM)
q = q_gate_heads[..., :HEAD_DIM]
gate_ref = q_gate_heads[..., HEAD_DIM:].reshape(num_tokens, num_q_heads * HEAD_DIM)
q_ref = q_norm.forward_native(q)
k_ref = k_norm.forward_native(k.view(num_tokens, num_kv_heads, HEAD_DIM))
assert isinstance(q_ref, torch.Tensor)
assert isinstance(k_ref, torch.Tensor)
q_ref = q_ref.reshape(num_tokens, num_q_heads * HEAD_DIM)
k_ref = k_ref.reshape(num_tokens, num_kv_heads * HEAD_DIM)
if mrope_section is None:
rope = RotaryEmbedding(
HEAD_DIM,
ROTARY_DIM,
MAX_POSITION_EMBEDDINGS,
ROPE_THETA,
True,
DTYPE,
).to(device)
positions = torch.arange(num_tokens, dtype=torch.long, device=device)
else:
rope = MRotaryEmbedding(
HEAD_DIM,
ROTARY_DIM,
MAX_POSITION_EMBEDDINGS,
ROPE_THETA,
True,
DTYPE,
mrope_section=list(mrope_section),
mrope_interleaved=True,
).to(device)
positions = torch.arange(3 * num_tokens, dtype=torch.long, device=device).view(
3, num_tokens
)
assert torch.unique(positions[:, 0]).numel() == 3
q_ref, k_ref = rope.forward_native(positions, q_ref, k_ref)
assert k_ref is not None
q_ref, k_ref, gate_ref = _ref_qk_rmsnorm_rope_gate(
q_gate,
k,
q_gamma,
k_gamma,
rope.cos_sin_cache,
positions,
RMS_NORM_EPS,
NUM_Q_HEADS,
NUM_KV_HEADS,
HEAD_DIM,
ROTARY_DIM,
)
q_out, k_out, gate_out = fused_qk_rmsnorm_rope_gate(
q_gate,
k,
q_gamma,
k_gamma,
q_norm.weight,
k_norm.weight,
rope.cos_sin_cache,
positions,
RMS_NORM_EPS,
NUM_Q_HEADS,
NUM_KV_HEADS,
num_q_heads,
num_kv_heads,
HEAD_DIM,
ROTARY_DIM,
mrope_section=mrope_section,
norm_beta=1.0,
)
atol, rtol = 2e-3, 5e-3
torch.testing.assert_close(q_out, q_ref, atol=atol, rtol=rtol)
torch.testing.assert_close(k_out, k_ref, atol=atol, rtol=rtol)
# gate is a verbatim copy of the source slice — must match bit-exactly.
# The built-in reference performs RoPE in BF16, while the fused kernel
# promotes the BF16-normalized values to FP32 for RoPE before storing BF16.
torch.testing.assert_close(q_out, q_ref, atol=1e-2, rtol=1e-2)
torch.testing.assert_close(k_out, k_ref, atol=1e-2, rtol=1e-2)
torch.testing.assert_close(gate_out, gate_ref, atol=0, rtol=0)
@@ -30,16 +30,22 @@ def _fused_qk_rmsnorm_rope_gate_kernel(
k_out_stride_t,
gate_out_stride_t,
cache_stride_p,
positions_stride_m,
positions_stride_t,
num_q_heads: tl.constexpr,
num_kv_heads: tl.constexpr,
head_dim: tl.constexpr,
rotary_dim: tl.constexpr,
half_rotary: tl.constexpr,
eps: tl.constexpr,
norm_beta: tl.constexpr,
INPUT_DTYPE: tl.constexpr,
HEAD_BLOCK: tl.constexpr,
ROT_HALF_BLOCK: tl.constexpr,
HAS_PASS: tl.constexpr,
HAS_MROPE: tl.constexpr,
MROPE_SECTION_H: tl.constexpr,
MROPE_SECTION_W: tl.constexpr,
):
token = tl.program_id(0)
head = tl.program_id(1)
@@ -61,7 +67,7 @@ def _fused_qk_rmsnorm_rope_gate_kernel(
x = tl.load(in_base + head_offs, mask=head_mask, other=0.0).to(tl.float32)
var = tl.sum(x * x, axis=0) / head_dim
inv_rms = tl.rsqrt(var + eps)
w = tl.load(w_ptr + head_offs, mask=head_mask, other=0.0).to(tl.float32)
w = tl.load(w_ptr + head_offs, mask=head_mask, other=0.0).to(tl.float32) + norm_beta
# Round-trip through INPUT_DTYPE so the RoPE input matches the bf16-storage
# behavior of the unfused (qk_rmsnorm -> memory -> apply_rope) reference path.
x_norm = (x * inv_rms * w).to(INPUT_DTYPE).to(tl.float32)
@@ -82,15 +88,30 @@ def _fused_qk_rmsnorm_rope_gate_kernel(
x_rot2 = tl.load(in_base + half_rotary + rot_offs, mask=rot_mask, other=0.0).to(
tl.float32
)
w_rot1 = tl.load(w_ptr + rot_offs, mask=rot_mask, other=0.0).to(tl.float32)
w_rot2 = tl.load(w_ptr + half_rotary + rot_offs, mask=rot_mask, other=0.0).to(
tl.float32
w_rot1 = (
tl.load(w_ptr + rot_offs, mask=rot_mask, other=0.0).to(tl.float32) + norm_beta
)
w_rot2 = (
tl.load(w_ptr + half_rotary + rot_offs, mask=rot_mask, other=0.0).to(tl.float32)
+ norm_beta
)
x_rot1 = (x_rot1 * inv_rms * w_rot1).to(INPUT_DTYPE).to(tl.float32)
x_rot2 = (x_rot2 * inv_rms * w_rot2).to(INPUT_DTYPE).to(tl.float32)
# Always use int64 for position to avoid overflow in address computation.
pos = tl.load(positions_ptr + token).to(tl.int64)
# Always use int64 for positions to avoid overflow in address computation.
pos_t = tl.load(positions_ptr + token * positions_stride_t).to(tl.int64)
if HAS_MROPE:
pos_h = tl.load(
positions_ptr + positions_stride_m + token * positions_stride_t
).to(tl.int64)
pos_w = tl.load(
positions_ptr + 2 * positions_stride_m + token * positions_stride_t
).to(tl.int64)
is_h = (rot_offs % 3 == 1) & (rot_offs < 3 * MROPE_SECTION_H)
is_w = (rot_offs % 3 == 2) & (rot_offs < 3 * MROPE_SECTION_W)
pos = tl.where(is_h, pos_h, tl.where(is_w, pos_w, pos_t))
else:
pos = pos_t
cache_offset = pos * cache_stride_p
cos = tl.load(
cos_sin_cache_ptr + cache_offset + rot_offs, mask=rot_mask, other=0.0
@@ -126,21 +147,25 @@ def fused_qk_rmsnorm_rope_gate(
num_kv_heads: int,
head_dim: int,
rotary_dim: int,
mrope_section: list[int] | tuple[int, int, int] | None = None,
norm_beta: float = 0.0,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
"""Fused split + QK-RMSNorm + (partial) RoPE + gate copy for Qwen3.5 attn.
"""Fused split + QK-RMSNorm + (partial) RoPE + gate copy for Qwen attn.
Args:
q_gate: (n_tokens, num_q_heads * 2 * head_dim) -- per head: [q|gate]
k: (n_tokens, num_kv_heads * head_dim)
q_weight: (head_dim,) GemmaRMSNorm effective weight (already +1)
k_weight: (head_dim,) GemmaRMSNorm effective weight (already +1)
q_weight: (head_dim,) RMSNorm weight
k_weight: (head_dim,) RMSNorm weight
cos_sin_cache: (max_pos, rotary_dim) packed [cos|sin]
positions: (n_tokens,) int32 or int64
positions: (n_tokens,) or (3, n_tokens) int32 or int64
eps: RMSNorm epsilon
num_q_heads: number of Q heads (after TP split)
num_kv_heads: number of KV heads (after TP split)
head_dim: per-head dimension
rotary_dim: rotary dimension; must be even and <= head_dim
mrope_section: interleaved T/H/W frequency counts for 2D positions
norm_beta: scalar added to the RMSNorm weight
Returns:
(q_out, k_out, gate_out) -- all contiguous (n_tokens, heads * head_dim).
@@ -151,6 +176,52 @@ def fused_qk_rmsnorm_rope_gate(
f"rotary_dim must be a positive even integer <= head_dim, "
f"got rotary_dim={rotary_dim}, head_dim={head_dim}"
)
if q_gate.dtype not in (torch.float16, torch.bfloat16) or k.dtype != q_gate.dtype:
raise ValueError(
"q_gate and k must have the same FP16 or BF16 dtype, "
f"got {q_gate.dtype} and {k.dtype}"
)
for name, tensor in (
("q_gate", q_gate),
("k", k),
("q_weight", q_weight),
("k_weight", k_weight),
("cos_sin_cache", cos_sin_cache),
):
if tensor.stride(-1) != 1:
raise ValueError(f"{name} must be contiguous in its last dimension")
if positions.ndim not in (1, 2):
raise ValueError(f"positions must be 1D or 2D, got shape={positions.shape}")
if positions.shape[-1] != q_gate.shape[0]:
raise ValueError(
"positions token dimension must match q_gate, "
f"got {positions.shape[-1]} and {q_gate.shape[0]}"
)
has_mrope = positions.ndim == 2
if has_mrope:
if positions.shape[0] != 3:
raise ValueError(
f"MRoPE positions must have shape (3, n_tokens), got {positions.shape}"
)
if mrope_section is None or len(mrope_section) != 3:
raise ValueError("mrope_section must contain the T/H/W frequency counts")
if sum(mrope_section) != rotary_dim // 2:
raise ValueError(
"mrope_section must sum to rotary_dim // 2, "
f"got {mrope_section} and rotary_dim={rotary_dim}"
)
mrope_section_h = mrope_section[1]
mrope_section_w = mrope_section[2]
positions_stride_m, positions_stride_t = positions.stride()
else:
if mrope_section is not None:
raise ValueError("mrope_section requires 2D MRoPE positions")
mrope_section_h = 0
mrope_section_w = 0
positions_stride_m = 0
positions_stride_t = positions.stride(0)
n_tokens = q_gate.shape[0]
q_out = torch.empty(
@@ -185,16 +256,22 @@ def fused_qk_rmsnorm_rope_gate(
k_out.stride(0),
gate_out.stride(0),
cos_sin_cache.stride(0),
positions_stride_m,
positions_stride_t,
num_q_heads,
num_kv_heads,
head_dim,
rotary_dim,
half_rotary,
eps,
norm_beta=norm_beta,
INPUT_DTYPE=tl.bfloat16 if q_gate.dtype == torch.bfloat16 else tl.float16,
HEAD_BLOCK=head_block,
ROT_HALF_BLOCK=rot_half_block,
HAS_PASS=rotary_dim < head_dim,
HAS_MROPE=has_mrope,
MROPE_SECTION_H=mrope_section_h,
MROPE_SECTION_W=mrope_section_w,
num_warps=num_warps,
num_stages=2,
)
+28 -10
View File
@@ -43,7 +43,7 @@ from vllm.model_executor.layers.mamba.mamba_utils import (
MambaStateShapeCalculator,
)
from vllm.model_executor.layers.quantization import QuantizationConfig
from vllm.model_executor.layers.rotary_embedding import get_rope
from vllm.model_executor.layers.rotary_embedding import MRotaryEmbedding, get_rope
from vllm.model_executor.layers.vocab_parallel_embedding import (
ParallelLMHead,
VocabParallelEmbedding,
@@ -300,14 +300,26 @@ class Qwen3NextAttention(nn.Module):
self.k_norm = Qwen3NextRMSNorm(self.head_dim, eps=config.rms_norm_eps)
# Fuse the gated split + QK-RMSNorm + (partial) NeoX RoPE + gate copy.
# TODO: support MRoPE
mm_config = model_config.multimodal_config if model_config else None
text_only = mm_config is None or mm_config.language_model_only
mrope_section = getattr(self.rotary_emb, "mrope_section", None)
supports_mrope = bool(
type(self.rotary_emb) is MRotaryEmbedding
and mrope_section
and len(mrope_section) == 3
and sum(mrope_section) == self.rotary_emb.rotary_dim // 2
and getattr(self.rotary_emb, "mrope_interleaved", False)
)
supports_dtype = getattr(self.rotary_emb, "dtype", None) in (
torch.float16,
torch.bfloat16,
)
self.use_fused_qk_norm_rope_gate = (
self.attn_output_gate
and getattr(self.rotary_emb, "is_neox_style", False)
and current_platform.is_cuda()
and text_only
and supports_dtype
and (text_only or supports_mrope)
)
def _project_qkv_gate(
@@ -325,22 +337,28 @@ class Qwen3NextAttention(nn.Module):
q_gate, k, v = qkv.split(
[self.q_size * 2, self.kv_size, self.kv_size], dim=-1
)
# mRoPE passes positions as (3, n_tokens) for T/H/W. Fusion is only
# enabled text-only, where the three rows are identical, so taking
# the T row is exact. (1D positions pass through.)
pos = positions[0] if positions.ndim == 2 else positions
if positions.ndim == 2 and not getattr(
self.rotary_emb, "mrope_section", None
):
positions = positions[0]
q, k, gate = fused_qk_rmsnorm_rope_gate(
q_gate,
k,
self.q_norm.weight.float() + 1.0,
self.k_norm.weight.float() + 1.0,
self.q_norm.weight,
self.k_norm.weight,
self.rotary_emb.cos_sin_cache,
pos,
positions,
self.q_norm.variance_epsilon,
self.num_heads,
self.num_kv_heads,
self.head_dim,
self.rotary_emb.rotary_dim,
norm_beta=1.0,
mrope_section=(
getattr(self.rotary_emb, "mrope_section", None)
if positions.ndim == 2
else None
),
)
return q, k, v, gate