[Kernel][Perf] Enable fused QK-norm + partial MRoPE + gate for Qwen3.6 (#52676)
Signed-off-by: kiroxu <148877251+BabyDrangoner@users.noreply.github.com> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
@@ -5,155 +5,114 @@ import pytest
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.layers.fused_qk_norm_rope import fused_qk_rmsnorm_rope_gate
|
||||
from vllm.model_executor.layers.rotary_embedding import RotaryEmbedding
|
||||
from vllm.model_executor.layers.layernorm import GemmaRMSNorm
|
||||
from vllm.model_executor.layers.rotary_embedding import (
|
||||
MRotaryEmbedding,
|
||||
RotaryEmbedding,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
|
||||
# Qwen/Qwen3.6-27B config (huggingface.co/Qwen/Qwen3.6-27B), TP=1 shapes.
|
||||
NUM_Q_HEADS = 24
|
||||
NUM_KV_HEADS = 4
|
||||
# Qwen3.6 TP=1 attention geometry.
|
||||
HEAD_DIM = 256
|
||||
PARTIAL_ROTARY_FACTOR = 0.25
|
||||
ROTARY_DIM = int(HEAD_DIM * PARTIAL_ROTARY_FACTOR) # 64
|
||||
ROTARY_DIM = 64
|
||||
RMS_NORM_EPS = 1e-6
|
||||
MAX_POSITION_EMBEDDINGS = 262144
|
||||
ROPE_THETA = 10000000.0
|
||||
|
||||
DTYPES = [torch.bfloat16]
|
||||
SEEDS = [13]
|
||||
NUM_TOKENS = [1, 4, 37]
|
||||
|
||||
|
||||
def _ref_qk_rmsnorm_rope_gate(
|
||||
q_gate: torch.Tensor,
|
||||
k: torch.Tensor,
|
||||
q_gamma: torch.Tensor,
|
||||
k_gamma: torch.Tensor,
|
||||
cos_sin_cache: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
eps: float,
|
||||
num_q_heads: int,
|
||||
num_kv_heads: int,
|
||||
head_dim: int,
|
||||
rotary_dim: int,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
"""PyTorch reference: split + RMSNorm + partial NeoX RoPE + gate extraction.
|
||||
|
||||
Matches ``fused_qk_rmsnorm_rope_gate``'s contract: ``q_gamma`` / ``k_gamma``
|
||||
are the already-adjusted effective gammas (for GemmaRMSNorm the caller
|
||||
has done ``weight + 1`` before passing them in).
|
||||
"""
|
||||
n_tokens = q_gate.shape[0]
|
||||
half = rotary_dim // 2
|
||||
|
||||
# Per head the q projection is laid out as [q | gate].
|
||||
q_gate = q_gate.view(n_tokens, num_q_heads, 2 * head_dim)
|
||||
q = q_gate[..., :head_dim]
|
||||
gate = q_gate[..., head_dim:].reshape(n_tokens, num_q_heads * head_dim)
|
||||
k = k.view(n_tokens, num_kv_heads, head_dim)
|
||||
|
||||
def rms_norm(x: torch.Tensor, gamma: torch.Tensor) -> torch.Tensor:
|
||||
orig_dtype = x.dtype
|
||||
x = x.float()
|
||||
var = x.pow(2).mean(dim=-1, keepdim=True)
|
||||
x = x * torch.rsqrt(var + eps)
|
||||
return (x * gamma.float()).to(orig_dtype)
|
||||
|
||||
q = rms_norm(q, q_gamma)
|
||||
k = rms_norm(k, k_gamma)
|
||||
|
||||
# Partial NeoX RoPE on the first ``rotary_dim`` elements of each head;
|
||||
# cos_sin_cache row is packed as [cos(half) | sin(half)].
|
||||
pos = positions.view(-1)
|
||||
cos = cos_sin_cache[pos, :half].float()[:, None, :]
|
||||
sin = cos_sin_cache[pos, half:rotary_dim].float()[:, None, :]
|
||||
|
||||
def rope(x: torch.Tensor) -> torch.Tensor:
|
||||
x_rot, x_pass = x[..., :rotary_dim], x[..., rotary_dim:]
|
||||
x1 = x_rot[..., :half].float()
|
||||
x2 = x_rot[..., half:].float()
|
||||
o1 = x1 * cos - x2 * sin
|
||||
o2 = x2 * cos + x1 * sin
|
||||
rotated = torch.cat([o1, o2], dim=-1).to(x.dtype)
|
||||
return torch.cat([rotated, x_pass], dim=-1)
|
||||
|
||||
q = rope(q).reshape(n_tokens, num_q_heads * head_dim)
|
||||
k = rope(k).reshape(n_tokens, num_kv_heads * head_dim)
|
||||
return q, k, gate
|
||||
DTYPE = torch.bfloat16
|
||||
SEED = 13
|
||||
MROPE_SECTION = (11, 11, 10)
|
||||
ROPE_CASES = [
|
||||
pytest.param(24, 4, None, id="rope"),
|
||||
pytest.param(16, 2, MROPE_SECTION, id="interleaved-mrope"),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda_alike(),
|
||||
reason="fused_qk_rmsnorm_rope_gate Triton kernel requires CUDA/ROCm",
|
||||
)
|
||||
@pytest.mark.parametrize("dtype", DTYPES)
|
||||
@pytest.mark.parametrize("seed", SEEDS)
|
||||
@pytest.mark.parametrize("num_tokens", NUM_TOKENS)
|
||||
@pytest.mark.parametrize("num_q_heads,num_kv_heads,mrope_section", ROPE_CASES)
|
||||
@pytest.mark.parametrize("num_tokens", [1, 4, 37])
|
||||
@torch.inference_mode()
|
||||
def test_fused_qk_norm_rope_gate_matches_reference(
|
||||
default_vllm_config,
|
||||
dtype: torch.dtype,
|
||||
seed: int,
|
||||
num_tokens: int,
|
||||
):
|
||||
num_q_heads: int,
|
||||
num_kv_heads: int,
|
||||
mrope_section: tuple[int, int, int] | None,
|
||||
) -> None:
|
||||
device = torch.device("cuda", torch.accelerator.current_device_index())
|
||||
torch.set_default_device(device)
|
||||
set_random_seed(seed)
|
||||
set_random_seed(SEED)
|
||||
|
||||
q_gate = torch.randn(
|
||||
num_tokens, NUM_Q_HEADS * 2 * HEAD_DIM, dtype=dtype, device=device
|
||||
)
|
||||
k = torch.randn(num_tokens, NUM_KV_HEADS * HEAD_DIM, dtype=dtype, device=device)
|
||||
# GemmaRMSNorm-style: the kernel takes the effective gamma (weight + 1).
|
||||
q_gamma = (
|
||||
torch.empty(HEAD_DIM, dtype=dtype, device=device).normal_(mean=0.0, std=0.1)
|
||||
+ 1.0
|
||||
)
|
||||
k_gamma = (
|
||||
torch.empty(HEAD_DIM, dtype=dtype, device=device).normal_(mean=0.0, std=0.1)
|
||||
+ 1.0
|
||||
num_tokens, num_q_heads * 2 * HEAD_DIM, dtype=DTYPE, device=device
|
||||
)
|
||||
k = torch.randn(num_tokens, num_kv_heads * HEAD_DIM, dtype=DTYPE, device=device)
|
||||
|
||||
# fused_qk_rmsnorm_rope_gate only handles NeoX-style RoPE.
|
||||
rope = RotaryEmbedding(
|
||||
head_size=HEAD_DIM,
|
||||
rotary_dim=ROTARY_DIM,
|
||||
max_position_embeddings=MAX_POSITION_EMBEDDINGS,
|
||||
base=ROPE_THETA,
|
||||
is_neox_style=True,
|
||||
dtype=dtype,
|
||||
).to(device)
|
||||
positions = torch.arange(num_tokens, dtype=torch.long, device=device)
|
||||
q_norm = GemmaRMSNorm(HEAD_DIM, eps=RMS_NORM_EPS).to(device, dtype=DTYPE)
|
||||
k_norm = GemmaRMSNorm(HEAD_DIM, eps=RMS_NORM_EPS).to(device, dtype=DTYPE)
|
||||
q_norm.weight.normal_(std=0.1)
|
||||
k_norm.weight.normal_(std=0.1)
|
||||
|
||||
q_gate_heads = q_gate.view(num_tokens, num_q_heads, 2 * HEAD_DIM)
|
||||
q = q_gate_heads[..., :HEAD_DIM]
|
||||
gate_ref = q_gate_heads[..., HEAD_DIM:].reshape(num_tokens, num_q_heads * HEAD_DIM)
|
||||
q_ref = q_norm.forward_native(q)
|
||||
k_ref = k_norm.forward_native(k.view(num_tokens, num_kv_heads, HEAD_DIM))
|
||||
assert isinstance(q_ref, torch.Tensor)
|
||||
assert isinstance(k_ref, torch.Tensor)
|
||||
q_ref = q_ref.reshape(num_tokens, num_q_heads * HEAD_DIM)
|
||||
k_ref = k_ref.reshape(num_tokens, num_kv_heads * HEAD_DIM)
|
||||
|
||||
if mrope_section is None:
|
||||
rope = RotaryEmbedding(
|
||||
HEAD_DIM,
|
||||
ROTARY_DIM,
|
||||
MAX_POSITION_EMBEDDINGS,
|
||||
ROPE_THETA,
|
||||
True,
|
||||
DTYPE,
|
||||
).to(device)
|
||||
positions = torch.arange(num_tokens, dtype=torch.long, device=device)
|
||||
else:
|
||||
rope = MRotaryEmbedding(
|
||||
HEAD_DIM,
|
||||
ROTARY_DIM,
|
||||
MAX_POSITION_EMBEDDINGS,
|
||||
ROPE_THETA,
|
||||
True,
|
||||
DTYPE,
|
||||
mrope_section=list(mrope_section),
|
||||
mrope_interleaved=True,
|
||||
).to(device)
|
||||
positions = torch.arange(3 * num_tokens, dtype=torch.long, device=device).view(
|
||||
3, num_tokens
|
||||
)
|
||||
assert torch.unique(positions[:, 0]).numel() == 3
|
||||
|
||||
q_ref, k_ref = rope.forward_native(positions, q_ref, k_ref)
|
||||
assert k_ref is not None
|
||||
|
||||
q_ref, k_ref, gate_ref = _ref_qk_rmsnorm_rope_gate(
|
||||
q_gate,
|
||||
k,
|
||||
q_gamma,
|
||||
k_gamma,
|
||||
rope.cos_sin_cache,
|
||||
positions,
|
||||
RMS_NORM_EPS,
|
||||
NUM_Q_HEADS,
|
||||
NUM_KV_HEADS,
|
||||
HEAD_DIM,
|
||||
ROTARY_DIM,
|
||||
)
|
||||
q_out, k_out, gate_out = fused_qk_rmsnorm_rope_gate(
|
||||
q_gate,
|
||||
k,
|
||||
q_gamma,
|
||||
k_gamma,
|
||||
q_norm.weight,
|
||||
k_norm.weight,
|
||||
rope.cos_sin_cache,
|
||||
positions,
|
||||
RMS_NORM_EPS,
|
||||
NUM_Q_HEADS,
|
||||
NUM_KV_HEADS,
|
||||
num_q_heads,
|
||||
num_kv_heads,
|
||||
HEAD_DIM,
|
||||
ROTARY_DIM,
|
||||
mrope_section=mrope_section,
|
||||
norm_beta=1.0,
|
||||
)
|
||||
|
||||
atol, rtol = 2e-3, 5e-3
|
||||
torch.testing.assert_close(q_out, q_ref, atol=atol, rtol=rtol)
|
||||
torch.testing.assert_close(k_out, k_ref, atol=atol, rtol=rtol)
|
||||
# gate is a verbatim copy of the source slice — must match bit-exactly.
|
||||
# The built-in reference performs RoPE in BF16, while the fused kernel
|
||||
# promotes the BF16-normalized values to FP32 for RoPE before storing BF16.
|
||||
torch.testing.assert_close(q_out, q_ref, atol=1e-2, rtol=1e-2)
|
||||
torch.testing.assert_close(k_out, k_ref, atol=1e-2, rtol=1e-2)
|
||||
torch.testing.assert_close(gate_out, gate_ref, atol=0, rtol=0)
|
||||
|
||||
Reference in New Issue
Block a user