[Kernel] Warm up Qwen GDN gated RMSNorm (#54251)

Signed-off-by: Zupeng Wang <71580390+zupengwang@users.noreply.github.com>
Co-authored-by: OpenAI <support@openai.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Zupeng Wang
2026-09-03 18:19:48 +08:00
committed by GitHub
co-authored by OpenAI mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 4cc0cb6f76
commit c21751c90b
3 changed files with 361 additions and 26 deletions
@@ -10,6 +10,7 @@ import torch
from vllm.logger import init_logger
if TYPE_CHECKING:
from vllm.config import ModelConfig
from vllm.v1.worker.gpu_model_runner import GPUModelRunner
logger = init_logger(__name__)
@@ -37,6 +38,9 @@ class _QwenGDNWarmupConfig:
conv_kernel_size: int
conv_state: torch.Tensor
conv_dtype: torch.dtype
norm_weight_dtype: torch.dtype
norm_before_gate: bool
norm_activation: str
a_log: torch.Tensor
dt_bias: torch.Tensor
state_stride_token: int
@@ -62,6 +66,7 @@ def _is_qwen_gdn_layer(module: object) -> bool:
"conv_kernel_size",
"tp_size",
"kv_cache",
"norm",
"A_log",
"dt_bias",
)
@@ -112,6 +117,7 @@ def _qwen_gdn_warmup_config(
tp_size = int(layer.tp_size)
h = int(layer.num_k_heads) // tp_size
hv = int(layer.num_v_heads) // tp_size
norm = layer.norm
return _QwenGDNWarmupConfig(
h=h,
@@ -121,6 +127,9 @@ def _qwen_gdn_warmup_config(
conv_kernel_size=int(layer.conv_kernel_size),
conv_state=conv_state,
conv_dtype=conv_state.dtype,
norm_weight_dtype=norm.weight.dtype,
norm_before_gate=bool(norm.norm_before_gate),
norm_activation=str(norm.activation),
a_log=layer.A_log,
dt_bias=layer.dt_bias,
state_stride_token=int(ssm_state.stride(0)),
@@ -134,6 +143,29 @@ def _qwen_gdn_warmup_config(
return None
def _warm_gated_rms_norm_kernel(
device: torch.device,
config: _QwenGDNWarmupConfig,
max_num_tokens: int,
x_dtype: torch.dtype,
) -> None:
from vllm.third_party.flash_linear_attention.ops.layernorm_guard import (
warmup_layer_norm_fwd,
)
warmup_layer_norm_fwd(
max_num_tokens=max_num_tokens,
rows_per_token=config.hv,
group_size=config.v,
x_dtype=x_dtype,
weight_dtype=config.norm_weight_dtype,
device=device,
norm_before_gate=config.norm_before_gate,
is_rms_norm=True,
activation=config.norm_activation,
)
def _warm_causal_conv1d_fwd_kernel(
device: torch.device, config: _QwenGDNWarmupConfig
) -> None:
@@ -249,7 +281,7 @@ def _synchronize_device(device: torch.device) -> None:
@torch.inference_mode()
def qwen_triton_warmup(
runner: "GPUModelRunner",
model_config: object,
model_config: "ModelConfig",
) -> None:
"""Warm Qwen Triton kernels reported by the JIT monitor."""
if runner.is_pooling_model:
@@ -275,6 +307,8 @@ def qwen_triton_warmup(
if gdn_config is None:
return
max_num_tokens = max(1, int(getattr(runner, "max_num_tokens", 1)))
_warm_gated_rms_norm_kernel(device, gdn_config, max_num_tokens, model_config.dtype)
_warm_causal_conv1d_fwd_kernel(device, gdn_config)
_warm_fused_post_conv_kernel(device, gdn_config)
_warm_fused_sigmoid_gating_delta_rule_update_kernel(device, gdn_config)