[Kernel] Warm up Qwen GDN gated RMSNorm (#54251)
Signed-off-by: Zupeng Wang <71580390+zupengwang@users.noreply.github.com> Co-authored-by: OpenAI <support@openai.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
OpenAI
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
4cc0cb6f76
commit
c21751c90b
@@ -7,10 +7,13 @@ import torch.nn.functional as F
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.third_party.flash_linear_attention.ops.layernorm_guard import (
|
||||
LayerNormFwdKernel,
|
||||
calc_rows_per_block,
|
||||
layer_norm_fwd,
|
||||
layernorm_fn,
|
||||
rms_norm_ref,
|
||||
)
|
||||
from vllm.triton_utils import triton
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
|
||||
DEVICE = "xpu" if current_platform.is_xpu() else "cuda"
|
||||
@@ -103,6 +106,59 @@ IS_RMS_NORM = [True, False]
|
||||
SEEDS = [0, 42]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("rows_per_token", [1, 2, 4, 8, 16])
|
||||
def test_layer_norm_fwd_warmup_keys_cover_qwen_gdn(
|
||||
rows_per_token: int,
|
||||
) -> None:
|
||||
device = torch.device(DEVICE)
|
||||
group_size = 128
|
||||
max_num_tokens = 512
|
||||
kernel = LayerNormFwdKernel()
|
||||
warmup_keys = set(
|
||||
kernel.get_warmup_keys(
|
||||
max_num_tokens=max_num_tokens,
|
||||
rows_per_token=rows_per_token,
|
||||
group_size=group_size,
|
||||
x_dtype=torch.bfloat16,
|
||||
weight_dtype=torch.bfloat16,
|
||||
device=device,
|
||||
norm_before_gate=True,
|
||||
is_rms_norm=True,
|
||||
activation="silu",
|
||||
)
|
||||
)
|
||||
|
||||
for num_tokens in range(1, max_num_tokens + 1):
|
||||
num_rows = num_tokens * rows_per_token
|
||||
runtime_key = kernel.dispatch(
|
||||
x_dtype=torch.bfloat16,
|
||||
y_dtype=torch.bfloat16,
|
||||
weight_dtype=torch.bfloat16,
|
||||
bias_dtype=None,
|
||||
z_dtype=torch.bfloat16,
|
||||
mean_dtype=None,
|
||||
rstd_dtype=torch.float32,
|
||||
x_aligned=True,
|
||||
y_aligned=True,
|
||||
weight_aligned=True,
|
||||
bias_aligned=True,
|
||||
z_aligned=True,
|
||||
mean_aligned=True,
|
||||
rstd_aligned=True,
|
||||
stride_x_row=group_size,
|
||||
stride_y_row=group_size,
|
||||
stride_z_row=group_size,
|
||||
M=num_rows,
|
||||
N=group_size,
|
||||
BLOCK_N=triton.next_power_of_2(group_size),
|
||||
ROWS_PER_BLOCK=calc_rows_per_block(num_rows, device),
|
||||
norm_before_gate=True,
|
||||
is_rms_norm=True,
|
||||
activation="silu",
|
||||
)
|
||||
assert runtime_key in warmup_keys
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_tokens", NUM_TOKENS)
|
||||
@pytest.mark.parametrize("hidden_size", HIDDEN_SIZES)
|
||||
@pytest.mark.parametrize("dtype", DTYPES)
|
||||
|
||||
Reference in New Issue
Block a user