[Kernel] Warm up Qwen GDN gated RMSNorm (#54251)

Signed-off-by: Zupeng Wang <71580390+zupengwang@users.noreply.github.com>
Co-authored-by: OpenAI <support@openai.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Zupeng Wang
2026-09-03 18:19:48 +08:00
committed by GitHub
co-authored by OpenAI mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 4cc0cb6f76
commit c21751c90b
3 changed files with 361 additions and 26 deletions
+56
View File
@@ -7,10 +7,13 @@ import torch.nn.functional as F
from vllm.platforms import current_platform
from vllm.third_party.flash_linear_attention.ops.layernorm_guard import (
LayerNormFwdKernel,
calc_rows_per_block,
layer_norm_fwd,
layernorm_fn,
rms_norm_ref,
)
from vllm.triton_utils import triton
from vllm.utils.torch_utils import set_random_seed
DEVICE = "xpu" if current_platform.is_xpu() else "cuda"
@@ -103,6 +106,59 @@ IS_RMS_NORM = [True, False]
SEEDS = [0, 42]
@pytest.mark.parametrize("rows_per_token", [1, 2, 4, 8, 16])
def test_layer_norm_fwd_warmup_keys_cover_qwen_gdn(
rows_per_token: int,
) -> None:
device = torch.device(DEVICE)
group_size = 128
max_num_tokens = 512
kernel = LayerNormFwdKernel()
warmup_keys = set(
kernel.get_warmup_keys(
max_num_tokens=max_num_tokens,
rows_per_token=rows_per_token,
group_size=group_size,
x_dtype=torch.bfloat16,
weight_dtype=torch.bfloat16,
device=device,
norm_before_gate=True,
is_rms_norm=True,
activation="silu",
)
)
for num_tokens in range(1, max_num_tokens + 1):
num_rows = num_tokens * rows_per_token
runtime_key = kernel.dispatch(
x_dtype=torch.bfloat16,
y_dtype=torch.bfloat16,
weight_dtype=torch.bfloat16,
bias_dtype=None,
z_dtype=torch.bfloat16,
mean_dtype=None,
rstd_dtype=torch.float32,
x_aligned=True,
y_aligned=True,
weight_aligned=True,
bias_aligned=True,
z_aligned=True,
mean_aligned=True,
rstd_aligned=True,
stride_x_row=group_size,
stride_y_row=group_size,
stride_z_row=group_size,
M=num_rows,
N=group_size,
BLOCK_N=triton.next_power_of_2(group_size),
ROWS_PER_BLOCK=calc_rows_per_block(num_rows, device),
norm_before_gate=True,
is_rms_norm=True,
activation="silu",
)
assert runtime_key in warmup_keys
@pytest.mark.parametrize("num_tokens", NUM_TOKENS)
@pytest.mark.parametrize("hidden_size", HIDDEN_SIZES)
@pytest.mark.parametrize("dtype", DTYPES)
@@ -10,6 +10,7 @@ import torch
from vllm.logger import init_logger
if TYPE_CHECKING:
from vllm.config import ModelConfig
from vllm.v1.worker.gpu_model_runner import GPUModelRunner
logger = init_logger(__name__)
@@ -37,6 +38,9 @@ class _QwenGDNWarmupConfig:
conv_kernel_size: int
conv_state: torch.Tensor
conv_dtype: torch.dtype
norm_weight_dtype: torch.dtype
norm_before_gate: bool
norm_activation: str
a_log: torch.Tensor
dt_bias: torch.Tensor
state_stride_token: int
@@ -62,6 +66,7 @@ def _is_qwen_gdn_layer(module: object) -> bool:
"conv_kernel_size",
"tp_size",
"kv_cache",
"norm",
"A_log",
"dt_bias",
)
@@ -112,6 +117,7 @@ def _qwen_gdn_warmup_config(
tp_size = int(layer.tp_size)
h = int(layer.num_k_heads) // tp_size
hv = int(layer.num_v_heads) // tp_size
norm = layer.norm
return _QwenGDNWarmupConfig(
h=h,
@@ -121,6 +127,9 @@ def _qwen_gdn_warmup_config(
conv_kernel_size=int(layer.conv_kernel_size),
conv_state=conv_state,
conv_dtype=conv_state.dtype,
norm_weight_dtype=norm.weight.dtype,
norm_before_gate=bool(norm.norm_before_gate),
norm_activation=str(norm.activation),
a_log=layer.A_log,
dt_bias=layer.dt_bias,
state_stride_token=int(ssm_state.stride(0)),
@@ -134,6 +143,29 @@ def _qwen_gdn_warmup_config(
return None
def _warm_gated_rms_norm_kernel(
device: torch.device,
config: _QwenGDNWarmupConfig,
max_num_tokens: int,
x_dtype: torch.dtype,
) -> None:
from vllm.third_party.flash_linear_attention.ops.layernorm_guard import (
warmup_layer_norm_fwd,
)
warmup_layer_norm_fwd(
max_num_tokens=max_num_tokens,
rows_per_token=config.hv,
group_size=config.v,
x_dtype=x_dtype,
weight_dtype=config.norm_weight_dtype,
device=device,
norm_before_gate=config.norm_before_gate,
is_rms_norm=True,
activation=config.norm_activation,
)
def _warm_causal_conv1d_fwd_kernel(
device: torch.device, config: _QwenGDNWarmupConfig
) -> None:
@@ -249,7 +281,7 @@ def _synchronize_device(device: torch.device) -> None:
@torch.inference_mode()
def qwen_triton_warmup(
runner: "GPUModelRunner",
model_config: object,
model_config: "ModelConfig",
) -> None:
"""Warm Qwen Triton kernels reported by the JIT monitor."""
if runner.is_pooling_model:
@@ -275,6 +307,8 @@ def qwen_triton_warmup(
if gdn_config is None:
return
max_num_tokens = max(1, int(getattr(runner, "max_num_tokens", 1)))
_warm_gated_rms_norm_kernel(device, gdn_config, max_num_tokens, model_config.dtype)
_warm_causal_conv1d_fwd_kernel(device, gdn_config)
_warm_fused_post_conv_kernel(device, gdn_config)
_warm_fused_sigmoid_gating_delta_rule_update_kernel(device, gdn_config)
+270 -25
View File
@@ -13,11 +13,19 @@
# This backward pass is faster for dimensions up to 8k, but after that it's much slower due to register spilling.
# The models we train have hidden dim up to 8k anyway (e.g. Llama 70B), so this is fine.
from dataclasses import dataclass
import torch
import torch.nn as nn
import torch.nn.functional as F
from einops import rearrange
from vllm.model_executor.warmup.jit_warmup import VllmJitKernel
from vllm.model_executor.warmup.jit_warmup_triton_helper import (
TritonPointerInputVariant,
TritonWarmupTensor,
triton_scalar_specialization_rep,
)
from vllm.triton_utils import tl, triton
from vllm.utils.math_utils import cdiv, next_power_of_2
from vllm.utils.platform_utils import num_compute_units
@@ -175,6 +183,263 @@ def calc_rows_per_block(M: int, device: torch.device) -> int:
return rows_per_block
class LayerNormFwdKernel(VllmJitKernel["LayerNormFwdKernel.CompileKey"]):
"""Own the runtime and compile-only dispatch for FLA layer norm."""
@dataclass(frozen=True)
class CompileKey:
x_dtype: torch.dtype
y_dtype: torch.dtype
weight_dtype: torch.dtype
bias_dtype: torch.dtype | None
z_dtype: torch.dtype | None
mean_dtype: torch.dtype | None
rstd_dtype: torch.dtype
input_variant: TritonPointerInputVariant
stride_x_row: int
stride_y_row: int
stride_z_row: int
M: int
N: int
BLOCK_N: int
ROWS_PER_BLOCK: int
NORM_BEFORE_GATE: bool
IS_RMS_NORM: bool
ACTIVATION: str
kernel = staticmethod(layer_norm_fwd_kernel)
def dispatch( # type: ignore[override]
self,
*,
x_dtype: torch.dtype,
y_dtype: torch.dtype,
weight_dtype: torch.dtype,
bias_dtype: torch.dtype | None,
z_dtype: torch.dtype | None,
mean_dtype: torch.dtype | None,
rstd_dtype: torch.dtype,
x_aligned: bool,
y_aligned: bool,
weight_aligned: bool,
bias_aligned: bool,
z_aligned: bool,
mean_aligned: bool,
rstd_aligned: bool,
stride_x_row: int,
stride_y_row: int,
stride_z_row: int,
M: int,
N: int,
BLOCK_N: int,
ROWS_PER_BLOCK: int,
norm_before_gate: bool,
is_rms_norm: bool,
activation: str,
) -> CompileKey:
input_variant = TritonPointerInputVariant.from_alignment(
x=x_aligned,
y=y_aligned,
weight=weight_aligned,
bias=bias_aligned,
z=z_aligned,
mean=mean_aligned,
rstd=rstd_aligned,
)
return self.CompileKey(
x_dtype=x_dtype,
y_dtype=y_dtype,
weight_dtype=weight_dtype,
bias_dtype=bias_dtype,
z_dtype=z_dtype,
mean_dtype=mean_dtype,
rstd_dtype=rstd_dtype,
input_variant=input_variant,
stride_x_row=triton_scalar_specialization_rep(stride_x_row),
stride_y_row=triton_scalar_specialization_rep(stride_y_row),
stride_z_row=triton_scalar_specialization_rep(stride_z_row),
M=triton_scalar_specialization_rep(M),
N=N,
BLOCK_N=BLOCK_N,
ROWS_PER_BLOCK=ROWS_PER_BLOCK,
NORM_BEFORE_GATE=norm_before_gate,
IS_RMS_NORM=is_rms_norm,
ACTIVATION=activation,
)
def get_warmup_keys(
self,
*,
max_num_tokens: int,
rows_per_token: int,
group_size: int,
x_dtype: torch.dtype,
weight_dtype: torch.dtype,
device: torch.device,
norm_before_gate: bool,
is_rms_norm: bool,
activation: str,
) -> list[CompileKey]:
if max_num_tokens < 1 or rows_per_token < 1:
return []
max_fused_size = 65536 // torch.empty((), dtype=x_dtype).element_size()
block_n = min(max_fused_size, triton.next_power_of_2(group_size))
if group_size > block_n:
return []
# Triton's runtime key only distinguishes three integer specialization
# classes. Keep one real M for each (M specialization, rows/block) pair.
m_values: dict[tuple[int, int], int] = {}
for num_tokens in range(1, max_num_tokens + 1):
M = num_tokens * rows_per_token
signature = (
triton_scalar_specialization_rep(M),
calc_rows_per_block(M, device),
)
m_values.setdefault(signature, M)
keys: dict[LayerNormFwdKernel.CompileKey, None] = {}
for M in m_values.values():
key = self.dispatch(
x_dtype=x_dtype,
y_dtype=x_dtype,
weight_dtype=weight_dtype,
bias_dtype=None,
z_dtype=x_dtype,
mean_dtype=None if is_rms_norm else torch.float32,
rstd_dtype=torch.float32,
x_aligned=True,
y_aligned=True,
weight_aligned=True,
bias_aligned=True,
z_aligned=True,
mean_aligned=True,
rstd_aligned=True,
stride_x_row=group_size,
stride_y_row=group_size,
stride_z_row=group_size,
M=M,
N=group_size,
BLOCK_N=block_n,
ROWS_PER_BLOCK=calc_rows_per_block(M, device),
norm_before_gate=norm_before_gate,
is_rms_norm=is_rms_norm,
activation=activation,
)
keys[key] = None
return list(keys)
def compile(self, compile_key: CompileKey) -> None:
warmup = getattr(self.kernel, "warmup", None)
assert warmup is not None
variant = compile_key.input_variant
def pointer(
name: str, dtype: torch.dtype | None
) -> TritonWarmupTensor | None:
return None if dtype is None else variant.pointer(name, dtype)
warmup(
pointer("x", compile_key.x_dtype),
pointer("y", compile_key.y_dtype),
pointer("weight", compile_key.weight_dtype),
pointer("bias", compile_key.bias_dtype),
pointer("z", compile_key.z_dtype),
pointer("mean", compile_key.mean_dtype),
pointer("rstd", compile_key.rstd_dtype),
compile_key.stride_x_row,
compile_key.stride_y_row,
compile_key.stride_z_row,
compile_key.M,
compile_key.N,
1e-5,
BLOCK_N=compile_key.BLOCK_N,
ROWS_PER_BLOCK=compile_key.ROWS_PER_BLOCK,
HAS_BIAS=compile_key.bias_dtype is not None,
HAS_Z=compile_key.z_dtype is not None,
NORM_BEFORE_GATE=compile_key.NORM_BEFORE_GATE,
IS_RMS_NORM=compile_key.IS_RMS_NORM,
ACTIVATION=compile_key.ACTIVATION,
num_warps=min(max(compile_key.BLOCK_N // 256, 1), 8),
grid=(1, 1),
)
def __call__(
self,
x: torch.Tensor,
out: torch.Tensor,
weight: torch.Tensor,
bias: torch.Tensor | None,
z: torch.Tensor | None,
mean: torch.Tensor | None,
rstd: torch.Tensor,
eps: float,
group_size: int,
norm_before_gate: bool,
is_rms_norm: bool,
activation: str,
) -> None:
M = x.shape[0]
max_fused_size = 65536 // x.element_size()
block_n = min(max_fused_size, triton.next_power_of_2(group_size))
if group_size > block_n:
raise RuntimeError("This layer norm doesn't support feature dim >= 64KB.")
rows_per_block = calc_rows_per_block(M, x.device)
grid = (cdiv(M, rows_per_block), x.shape[1] // group_size)
self.kernel[grid](
x,
out,
weight,
bias,
z,
mean,
rstd,
x.stride(0),
out.stride(0),
z.stride(0) if z is not None else 0,
M,
group_size,
eps,
BLOCK_N=block_n,
ROWS_PER_BLOCK=rows_per_block,
HAS_BIAS=bias is not None,
HAS_Z=z is not None,
NORM_BEFORE_GATE=norm_before_gate,
IS_RMS_NORM=is_rms_norm,
ACTIVATION=activation,
num_warps=min(max(block_n // 256, 1), 8),
)
_LAYER_NORM_FWD_KERNEL = LayerNormFwdKernel()
def warmup_layer_norm_fwd(
*,
max_num_tokens: int,
rows_per_token: int,
group_size: int,
x_dtype: torch.dtype,
weight_dtype: torch.dtype,
device: torch.device,
norm_before_gate: bool,
is_rms_norm: bool,
activation: str,
) -> None:
_LAYER_NORM_FWD_KERNEL.warmup(
max_num_tokens=max_num_tokens,
rows_per_token=rows_per_token,
group_size=group_size,
x_dtype=x_dtype,
weight_dtype=weight_dtype,
device=device,
norm_before_gate=norm_before_gate,
is_rms_norm=is_rms_norm,
activation=activation,
)
def layer_norm_fwd(
x: torch.Tensor,
weight: torch.Tensor,
@@ -213,18 +478,7 @@ def layer_norm_fwd(
else None
)
rstd = torch.empty((ngroups * M,), dtype=torch.float32, device=x.device)
# Less than 64KB per feature: enqueue fused kernel
MAX_FUSED_SIZE = 65536 // x.element_size()
BLOCK_N = min(MAX_FUSED_SIZE, triton.next_power_of_2(group_size))
if group_size > BLOCK_N:
raise RuntimeError("This layer norm doesn't support feature dim >= 64KB.")
# heuristics for number of warps
num_warps = min(max(BLOCK_N // 256, 1), 8)
# Calculate rows per block based on SM count
rows_per_block = calc_rows_per_block(M, x.device)
# Update grid to use rows_per_block
grid = (cdiv(M, rows_per_block), ngroups)
layer_norm_fwd_kernel[grid](
_LAYER_NORM_FWD_KERNEL(
x,
out,
weight,
@@ -232,20 +486,11 @@ def layer_norm_fwd(
z,
mean,
rstd,
x.stride(0),
out.stride(0),
z.stride(0) if z is not None else 0,
M,
group_size,
eps,
BLOCK_N=BLOCK_N,
ROWS_PER_BLOCK=rows_per_block,
HAS_BIAS=bias is not None,
HAS_Z=z is not None,
NORM_BEFORE_GATE=norm_before_gate,
IS_RMS_NORM=is_rms_norm,
num_warps=num_warps,
ACTIVATION=activation,
group_size,
norm_before_gate,
is_rms_norm,
activation,
)
return out, mean, rstd