From c21751c90b1b1d4cfd8fd6e951feaeb539171cfc Mon Sep 17 00:00:00 2001 From: Zupeng Wang Date: Thu, 3 Sep 2026 18:19:48 +0800 Subject: [PATCH] [Kernel] Warm up Qwen GDN gated RMSNorm (#54251) Signed-off-by: Zupeng Wang <71580390+zupengwang@users.noreply.github.com> Co-authored-by: OpenAI Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- tests/kernels/test_fla_layernorm_guard.py | 56 ++++ .../warmup/qwen_triton_warmup.py | 36 ++- .../ops/layernorm_guard.py | 295 ++++++++++++++++-- 3 files changed, 361 insertions(+), 26 deletions(-) diff --git a/tests/kernels/test_fla_layernorm_guard.py b/tests/kernels/test_fla_layernorm_guard.py index f669bd1814..492c099bf1 100644 --- a/tests/kernels/test_fla_layernorm_guard.py +++ b/tests/kernels/test_fla_layernorm_guard.py @@ -7,10 +7,13 @@ import torch.nn.functional as F from vllm.platforms import current_platform from vllm.third_party.flash_linear_attention.ops.layernorm_guard import ( + LayerNormFwdKernel, + calc_rows_per_block, layer_norm_fwd, layernorm_fn, rms_norm_ref, ) +from vllm.triton_utils import triton from vllm.utils.torch_utils import set_random_seed DEVICE = "xpu" if current_platform.is_xpu() else "cuda" @@ -103,6 +106,59 @@ IS_RMS_NORM = [True, False] SEEDS = [0, 42] +@pytest.mark.parametrize("rows_per_token", [1, 2, 4, 8, 16]) +def test_layer_norm_fwd_warmup_keys_cover_qwen_gdn( + rows_per_token: int, +) -> None: + device = torch.device(DEVICE) + group_size = 128 + max_num_tokens = 512 + kernel = LayerNormFwdKernel() + warmup_keys = set( + kernel.get_warmup_keys( + max_num_tokens=max_num_tokens, + rows_per_token=rows_per_token, + group_size=group_size, + x_dtype=torch.bfloat16, + weight_dtype=torch.bfloat16, + device=device, + norm_before_gate=True, + is_rms_norm=True, + activation="silu", + ) + ) + + for num_tokens in range(1, max_num_tokens + 1): + num_rows = num_tokens * rows_per_token + runtime_key = kernel.dispatch( + x_dtype=torch.bfloat16, + y_dtype=torch.bfloat16, + weight_dtype=torch.bfloat16, + bias_dtype=None, + z_dtype=torch.bfloat16, + mean_dtype=None, + rstd_dtype=torch.float32, + x_aligned=True, + y_aligned=True, + weight_aligned=True, + bias_aligned=True, + z_aligned=True, + mean_aligned=True, + rstd_aligned=True, + stride_x_row=group_size, + stride_y_row=group_size, + stride_z_row=group_size, + M=num_rows, + N=group_size, + BLOCK_N=triton.next_power_of_2(group_size), + ROWS_PER_BLOCK=calc_rows_per_block(num_rows, device), + norm_before_gate=True, + is_rms_norm=True, + activation="silu", + ) + assert runtime_key in warmup_keys + + @pytest.mark.parametrize("num_tokens", NUM_TOKENS) @pytest.mark.parametrize("hidden_size", HIDDEN_SIZES) @pytest.mark.parametrize("dtype", DTYPES) diff --git a/vllm/model_executor/warmup/qwen_triton_warmup.py b/vllm/model_executor/warmup/qwen_triton_warmup.py index 06a016a5a0..3b784b102e 100644 --- a/vllm/model_executor/warmup/qwen_triton_warmup.py +++ b/vllm/model_executor/warmup/qwen_triton_warmup.py @@ -10,6 +10,7 @@ import torch from vllm.logger import init_logger if TYPE_CHECKING: + from vllm.config import ModelConfig from vllm.v1.worker.gpu_model_runner import GPUModelRunner logger = init_logger(__name__) @@ -37,6 +38,9 @@ class _QwenGDNWarmupConfig: conv_kernel_size: int conv_state: torch.Tensor conv_dtype: torch.dtype + norm_weight_dtype: torch.dtype + norm_before_gate: bool + norm_activation: str a_log: torch.Tensor dt_bias: torch.Tensor state_stride_token: int @@ -62,6 +66,7 @@ def _is_qwen_gdn_layer(module: object) -> bool: "conv_kernel_size", "tp_size", "kv_cache", + "norm", "A_log", "dt_bias", ) @@ -112,6 +117,7 @@ def _qwen_gdn_warmup_config( tp_size = int(layer.tp_size) h = int(layer.num_k_heads) // tp_size hv = int(layer.num_v_heads) // tp_size + norm = layer.norm return _QwenGDNWarmupConfig( h=h, @@ -121,6 +127,9 @@ def _qwen_gdn_warmup_config( conv_kernel_size=int(layer.conv_kernel_size), conv_state=conv_state, conv_dtype=conv_state.dtype, + norm_weight_dtype=norm.weight.dtype, + norm_before_gate=bool(norm.norm_before_gate), + norm_activation=str(norm.activation), a_log=layer.A_log, dt_bias=layer.dt_bias, state_stride_token=int(ssm_state.stride(0)), @@ -134,6 +143,29 @@ def _qwen_gdn_warmup_config( return None +def _warm_gated_rms_norm_kernel( + device: torch.device, + config: _QwenGDNWarmupConfig, + max_num_tokens: int, + x_dtype: torch.dtype, +) -> None: + from vllm.third_party.flash_linear_attention.ops.layernorm_guard import ( + warmup_layer_norm_fwd, + ) + + warmup_layer_norm_fwd( + max_num_tokens=max_num_tokens, + rows_per_token=config.hv, + group_size=config.v, + x_dtype=x_dtype, + weight_dtype=config.norm_weight_dtype, + device=device, + norm_before_gate=config.norm_before_gate, + is_rms_norm=True, + activation=config.norm_activation, + ) + + def _warm_causal_conv1d_fwd_kernel( device: torch.device, config: _QwenGDNWarmupConfig ) -> None: @@ -249,7 +281,7 @@ def _synchronize_device(device: torch.device) -> None: @torch.inference_mode() def qwen_triton_warmup( runner: "GPUModelRunner", - model_config: object, + model_config: "ModelConfig", ) -> None: """Warm Qwen Triton kernels reported by the JIT monitor.""" if runner.is_pooling_model: @@ -275,6 +307,8 @@ def qwen_triton_warmup( if gdn_config is None: return + max_num_tokens = max(1, int(getattr(runner, "max_num_tokens", 1))) + _warm_gated_rms_norm_kernel(device, gdn_config, max_num_tokens, model_config.dtype) _warm_causal_conv1d_fwd_kernel(device, gdn_config) _warm_fused_post_conv_kernel(device, gdn_config) _warm_fused_sigmoid_gating_delta_rule_update_kernel(device, gdn_config) diff --git a/vllm/third_party/flash_linear_attention/ops/layernorm_guard.py b/vllm/third_party/flash_linear_attention/ops/layernorm_guard.py index 279b06b5eb..79f5abf031 100644 --- a/vllm/third_party/flash_linear_attention/ops/layernorm_guard.py +++ b/vllm/third_party/flash_linear_attention/ops/layernorm_guard.py @@ -13,11 +13,19 @@ # This backward pass is faster for dimensions up to 8k, but after that it's much slower due to register spilling. # The models we train have hidden dim up to 8k anyway (e.g. Llama 70B), so this is fine. +from dataclasses import dataclass + import torch import torch.nn as nn import torch.nn.functional as F from einops import rearrange +from vllm.model_executor.warmup.jit_warmup import VllmJitKernel +from vllm.model_executor.warmup.jit_warmup_triton_helper import ( + TritonPointerInputVariant, + TritonWarmupTensor, + triton_scalar_specialization_rep, +) from vllm.triton_utils import tl, triton from vllm.utils.math_utils import cdiv, next_power_of_2 from vllm.utils.platform_utils import num_compute_units @@ -175,6 +183,263 @@ def calc_rows_per_block(M: int, device: torch.device) -> int: return rows_per_block +class LayerNormFwdKernel(VllmJitKernel["LayerNormFwdKernel.CompileKey"]): + """Own the runtime and compile-only dispatch for FLA layer norm.""" + + @dataclass(frozen=True) + class CompileKey: + x_dtype: torch.dtype + y_dtype: torch.dtype + weight_dtype: torch.dtype + bias_dtype: torch.dtype | None + z_dtype: torch.dtype | None + mean_dtype: torch.dtype | None + rstd_dtype: torch.dtype + input_variant: TritonPointerInputVariant + stride_x_row: int + stride_y_row: int + stride_z_row: int + M: int + N: int + BLOCK_N: int + ROWS_PER_BLOCK: int + NORM_BEFORE_GATE: bool + IS_RMS_NORM: bool + ACTIVATION: str + + kernel = staticmethod(layer_norm_fwd_kernel) + + def dispatch( # type: ignore[override] + self, + *, + x_dtype: torch.dtype, + y_dtype: torch.dtype, + weight_dtype: torch.dtype, + bias_dtype: torch.dtype | None, + z_dtype: torch.dtype | None, + mean_dtype: torch.dtype | None, + rstd_dtype: torch.dtype, + x_aligned: bool, + y_aligned: bool, + weight_aligned: bool, + bias_aligned: bool, + z_aligned: bool, + mean_aligned: bool, + rstd_aligned: bool, + stride_x_row: int, + stride_y_row: int, + stride_z_row: int, + M: int, + N: int, + BLOCK_N: int, + ROWS_PER_BLOCK: int, + norm_before_gate: bool, + is_rms_norm: bool, + activation: str, + ) -> CompileKey: + input_variant = TritonPointerInputVariant.from_alignment( + x=x_aligned, + y=y_aligned, + weight=weight_aligned, + bias=bias_aligned, + z=z_aligned, + mean=mean_aligned, + rstd=rstd_aligned, + ) + return self.CompileKey( + x_dtype=x_dtype, + y_dtype=y_dtype, + weight_dtype=weight_dtype, + bias_dtype=bias_dtype, + z_dtype=z_dtype, + mean_dtype=mean_dtype, + rstd_dtype=rstd_dtype, + input_variant=input_variant, + stride_x_row=triton_scalar_specialization_rep(stride_x_row), + stride_y_row=triton_scalar_specialization_rep(stride_y_row), + stride_z_row=triton_scalar_specialization_rep(stride_z_row), + M=triton_scalar_specialization_rep(M), + N=N, + BLOCK_N=BLOCK_N, + ROWS_PER_BLOCK=ROWS_PER_BLOCK, + NORM_BEFORE_GATE=norm_before_gate, + IS_RMS_NORM=is_rms_norm, + ACTIVATION=activation, + ) + + def get_warmup_keys( + self, + *, + max_num_tokens: int, + rows_per_token: int, + group_size: int, + x_dtype: torch.dtype, + weight_dtype: torch.dtype, + device: torch.device, + norm_before_gate: bool, + is_rms_norm: bool, + activation: str, + ) -> list[CompileKey]: + if max_num_tokens < 1 or rows_per_token < 1: + return [] + + max_fused_size = 65536 // torch.empty((), dtype=x_dtype).element_size() + block_n = min(max_fused_size, triton.next_power_of_2(group_size)) + if group_size > block_n: + return [] + + # Triton's runtime key only distinguishes three integer specialization + # classes. Keep one real M for each (M specialization, rows/block) pair. + m_values: dict[tuple[int, int], int] = {} + for num_tokens in range(1, max_num_tokens + 1): + M = num_tokens * rows_per_token + signature = ( + triton_scalar_specialization_rep(M), + calc_rows_per_block(M, device), + ) + m_values.setdefault(signature, M) + + keys: dict[LayerNormFwdKernel.CompileKey, None] = {} + for M in m_values.values(): + key = self.dispatch( + x_dtype=x_dtype, + y_dtype=x_dtype, + weight_dtype=weight_dtype, + bias_dtype=None, + z_dtype=x_dtype, + mean_dtype=None if is_rms_norm else torch.float32, + rstd_dtype=torch.float32, + x_aligned=True, + y_aligned=True, + weight_aligned=True, + bias_aligned=True, + z_aligned=True, + mean_aligned=True, + rstd_aligned=True, + stride_x_row=group_size, + stride_y_row=group_size, + stride_z_row=group_size, + M=M, + N=group_size, + BLOCK_N=block_n, + ROWS_PER_BLOCK=calc_rows_per_block(M, device), + norm_before_gate=norm_before_gate, + is_rms_norm=is_rms_norm, + activation=activation, + ) + keys[key] = None + return list(keys) + + def compile(self, compile_key: CompileKey) -> None: + warmup = getattr(self.kernel, "warmup", None) + assert warmup is not None + variant = compile_key.input_variant + + def pointer( + name: str, dtype: torch.dtype | None + ) -> TritonWarmupTensor | None: + return None if dtype is None else variant.pointer(name, dtype) + + warmup( + pointer("x", compile_key.x_dtype), + pointer("y", compile_key.y_dtype), + pointer("weight", compile_key.weight_dtype), + pointer("bias", compile_key.bias_dtype), + pointer("z", compile_key.z_dtype), + pointer("mean", compile_key.mean_dtype), + pointer("rstd", compile_key.rstd_dtype), + compile_key.stride_x_row, + compile_key.stride_y_row, + compile_key.stride_z_row, + compile_key.M, + compile_key.N, + 1e-5, + BLOCK_N=compile_key.BLOCK_N, + ROWS_PER_BLOCK=compile_key.ROWS_PER_BLOCK, + HAS_BIAS=compile_key.bias_dtype is not None, + HAS_Z=compile_key.z_dtype is not None, + NORM_BEFORE_GATE=compile_key.NORM_BEFORE_GATE, + IS_RMS_NORM=compile_key.IS_RMS_NORM, + ACTIVATION=compile_key.ACTIVATION, + num_warps=min(max(compile_key.BLOCK_N // 256, 1), 8), + grid=(1, 1), + ) + + def __call__( + self, + x: torch.Tensor, + out: torch.Tensor, + weight: torch.Tensor, + bias: torch.Tensor | None, + z: torch.Tensor | None, + mean: torch.Tensor | None, + rstd: torch.Tensor, + eps: float, + group_size: int, + norm_before_gate: bool, + is_rms_norm: bool, + activation: str, + ) -> None: + M = x.shape[0] + max_fused_size = 65536 // x.element_size() + block_n = min(max_fused_size, triton.next_power_of_2(group_size)) + if group_size > block_n: + raise RuntimeError("This layer norm doesn't support feature dim >= 64KB.") + rows_per_block = calc_rows_per_block(M, x.device) + grid = (cdiv(M, rows_per_block), x.shape[1] // group_size) + self.kernel[grid]( + x, + out, + weight, + bias, + z, + mean, + rstd, + x.stride(0), + out.stride(0), + z.stride(0) if z is not None else 0, + M, + group_size, + eps, + BLOCK_N=block_n, + ROWS_PER_BLOCK=rows_per_block, + HAS_BIAS=bias is not None, + HAS_Z=z is not None, + NORM_BEFORE_GATE=norm_before_gate, + IS_RMS_NORM=is_rms_norm, + ACTIVATION=activation, + num_warps=min(max(block_n // 256, 1), 8), + ) + + +_LAYER_NORM_FWD_KERNEL = LayerNormFwdKernel() + + +def warmup_layer_norm_fwd( + *, + max_num_tokens: int, + rows_per_token: int, + group_size: int, + x_dtype: torch.dtype, + weight_dtype: torch.dtype, + device: torch.device, + norm_before_gate: bool, + is_rms_norm: bool, + activation: str, +) -> None: + _LAYER_NORM_FWD_KERNEL.warmup( + max_num_tokens=max_num_tokens, + rows_per_token=rows_per_token, + group_size=group_size, + x_dtype=x_dtype, + weight_dtype=weight_dtype, + device=device, + norm_before_gate=norm_before_gate, + is_rms_norm=is_rms_norm, + activation=activation, + ) + + def layer_norm_fwd( x: torch.Tensor, weight: torch.Tensor, @@ -213,18 +478,7 @@ def layer_norm_fwd( else None ) rstd = torch.empty((ngroups * M,), dtype=torch.float32, device=x.device) - # Less than 64KB per feature: enqueue fused kernel - MAX_FUSED_SIZE = 65536 // x.element_size() - BLOCK_N = min(MAX_FUSED_SIZE, triton.next_power_of_2(group_size)) - if group_size > BLOCK_N: - raise RuntimeError("This layer norm doesn't support feature dim >= 64KB.") - # heuristics for number of warps - num_warps = min(max(BLOCK_N // 256, 1), 8) - # Calculate rows per block based on SM count - rows_per_block = calc_rows_per_block(M, x.device) - # Update grid to use rows_per_block - grid = (cdiv(M, rows_per_block), ngroups) - layer_norm_fwd_kernel[grid]( + _LAYER_NORM_FWD_KERNEL( x, out, weight, @@ -232,20 +486,11 @@ def layer_norm_fwd( z, mean, rstd, - x.stride(0), - out.stride(0), - z.stride(0) if z is not None else 0, - M, - group_size, eps, - BLOCK_N=BLOCK_N, - ROWS_PER_BLOCK=rows_per_block, - HAS_BIAS=bias is not None, - HAS_Z=z is not None, - NORM_BEFORE_GATE=norm_before_gate, - IS_RMS_NORM=is_rms_norm, - num_warps=num_warps, - ACTIVATION=activation, + group_size, + norm_before_gate, + is_rms_norm, + activation, ) return out, mean, rstd