[Kernel] Warm up Qwen GDN gated RMSNorm (#54251)

Signed-off-by: Zupeng Wang <71580390+zupengwang@users.noreply.github.com>
Co-authored-by: OpenAI <support@openai.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Zupeng Wang
2026-09-03 18:19:48 +08:00
committed by GitHub
co-authored by OpenAI mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 4cc0cb6f76
commit c21751c90b
3 changed files with 361 additions and 26 deletions
+56
View File
@@ -7,10 +7,13 @@ import torch.nn.functional as F
from vllm.platforms import current_platform
from vllm.third_party.flash_linear_attention.ops.layernorm_guard import (
LayerNormFwdKernel,
calc_rows_per_block,
layer_norm_fwd,
layernorm_fn,
rms_norm_ref,
)
from vllm.triton_utils import triton
from vllm.utils.torch_utils import set_random_seed
DEVICE = "xpu" if current_platform.is_xpu() else "cuda"
@@ -103,6 +106,59 @@ IS_RMS_NORM = [True, False]
SEEDS = [0, 42]
@pytest.mark.parametrize("rows_per_token", [1, 2, 4, 8, 16])
def test_layer_norm_fwd_warmup_keys_cover_qwen_gdn(
rows_per_token: int,
) -> None:
device = torch.device(DEVICE)
group_size = 128
max_num_tokens = 512
kernel = LayerNormFwdKernel()
warmup_keys = set(
kernel.get_warmup_keys(
max_num_tokens=max_num_tokens,
rows_per_token=rows_per_token,
group_size=group_size,
x_dtype=torch.bfloat16,
weight_dtype=torch.bfloat16,
device=device,
norm_before_gate=True,
is_rms_norm=True,
activation="silu",
)
)
for num_tokens in range(1, max_num_tokens + 1):
num_rows = num_tokens * rows_per_token
runtime_key = kernel.dispatch(
x_dtype=torch.bfloat16,
y_dtype=torch.bfloat16,
weight_dtype=torch.bfloat16,
bias_dtype=None,
z_dtype=torch.bfloat16,
mean_dtype=None,
rstd_dtype=torch.float32,
x_aligned=True,
y_aligned=True,
weight_aligned=True,
bias_aligned=True,
z_aligned=True,
mean_aligned=True,
rstd_aligned=True,
stride_x_row=group_size,
stride_y_row=group_size,
stride_z_row=group_size,
M=num_rows,
N=group_size,
BLOCK_N=triton.next_power_of_2(group_size),
ROWS_PER_BLOCK=calc_rows_per_block(num_rows, device),
norm_before_gate=True,
is_rms_norm=True,
activation="silu",
)
assert runtime_key in warmup_keys
@pytest.mark.parametrize("num_tokens", NUM_TOKENS)
@pytest.mark.parametrize("hidden_size", HIDDEN_SIZES)
@pytest.mark.parametrize("dtype", DTYPES)