[Kernel] Warm up Qwen GDN gated RMSNorm (#54251)
Signed-off-by: Zupeng Wang <71580390+zupengwang@users.noreply.github.com> Co-authored-by: OpenAI <support@openai.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
OpenAI
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
4cc0cb6f76
commit
c21751c90b
@@ -7,10 +7,13 @@ import torch.nn.functional as F
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.third_party.flash_linear_attention.ops.layernorm_guard import (
|
||||
LayerNormFwdKernel,
|
||||
calc_rows_per_block,
|
||||
layer_norm_fwd,
|
||||
layernorm_fn,
|
||||
rms_norm_ref,
|
||||
)
|
||||
from vllm.triton_utils import triton
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
|
||||
DEVICE = "xpu" if current_platform.is_xpu() else "cuda"
|
||||
@@ -103,6 +106,59 @@ IS_RMS_NORM = [True, False]
|
||||
SEEDS = [0, 42]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("rows_per_token", [1, 2, 4, 8, 16])
|
||||
def test_layer_norm_fwd_warmup_keys_cover_qwen_gdn(
|
||||
rows_per_token: int,
|
||||
) -> None:
|
||||
device = torch.device(DEVICE)
|
||||
group_size = 128
|
||||
max_num_tokens = 512
|
||||
kernel = LayerNormFwdKernel()
|
||||
warmup_keys = set(
|
||||
kernel.get_warmup_keys(
|
||||
max_num_tokens=max_num_tokens,
|
||||
rows_per_token=rows_per_token,
|
||||
group_size=group_size,
|
||||
x_dtype=torch.bfloat16,
|
||||
weight_dtype=torch.bfloat16,
|
||||
device=device,
|
||||
norm_before_gate=True,
|
||||
is_rms_norm=True,
|
||||
activation="silu",
|
||||
)
|
||||
)
|
||||
|
||||
for num_tokens in range(1, max_num_tokens + 1):
|
||||
num_rows = num_tokens * rows_per_token
|
||||
runtime_key = kernel.dispatch(
|
||||
x_dtype=torch.bfloat16,
|
||||
y_dtype=torch.bfloat16,
|
||||
weight_dtype=torch.bfloat16,
|
||||
bias_dtype=None,
|
||||
z_dtype=torch.bfloat16,
|
||||
mean_dtype=None,
|
||||
rstd_dtype=torch.float32,
|
||||
x_aligned=True,
|
||||
y_aligned=True,
|
||||
weight_aligned=True,
|
||||
bias_aligned=True,
|
||||
z_aligned=True,
|
||||
mean_aligned=True,
|
||||
rstd_aligned=True,
|
||||
stride_x_row=group_size,
|
||||
stride_y_row=group_size,
|
||||
stride_z_row=group_size,
|
||||
M=num_rows,
|
||||
N=group_size,
|
||||
BLOCK_N=triton.next_power_of_2(group_size),
|
||||
ROWS_PER_BLOCK=calc_rows_per_block(num_rows, device),
|
||||
norm_before_gate=True,
|
||||
is_rms_norm=True,
|
||||
activation="silu",
|
||||
)
|
||||
assert runtime_key in warmup_keys
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_tokens", NUM_TOKENS)
|
||||
@pytest.mark.parametrize("hidden_size", HIDDEN_SIZES)
|
||||
@pytest.mark.parametrize("dtype", DTYPES)
|
||||
|
||||
@@ -10,6 +10,7 @@ import torch
|
||||
from vllm.logger import init_logger
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.v1.worker.gpu_model_runner import GPUModelRunner
|
||||
|
||||
logger = init_logger(__name__)
|
||||
@@ -37,6 +38,9 @@ class _QwenGDNWarmupConfig:
|
||||
conv_kernel_size: int
|
||||
conv_state: torch.Tensor
|
||||
conv_dtype: torch.dtype
|
||||
norm_weight_dtype: torch.dtype
|
||||
norm_before_gate: bool
|
||||
norm_activation: str
|
||||
a_log: torch.Tensor
|
||||
dt_bias: torch.Tensor
|
||||
state_stride_token: int
|
||||
@@ -62,6 +66,7 @@ def _is_qwen_gdn_layer(module: object) -> bool:
|
||||
"conv_kernel_size",
|
||||
"tp_size",
|
||||
"kv_cache",
|
||||
"norm",
|
||||
"A_log",
|
||||
"dt_bias",
|
||||
)
|
||||
@@ -112,6 +117,7 @@ def _qwen_gdn_warmup_config(
|
||||
tp_size = int(layer.tp_size)
|
||||
h = int(layer.num_k_heads) // tp_size
|
||||
hv = int(layer.num_v_heads) // tp_size
|
||||
norm = layer.norm
|
||||
|
||||
return _QwenGDNWarmupConfig(
|
||||
h=h,
|
||||
@@ -121,6 +127,9 @@ def _qwen_gdn_warmup_config(
|
||||
conv_kernel_size=int(layer.conv_kernel_size),
|
||||
conv_state=conv_state,
|
||||
conv_dtype=conv_state.dtype,
|
||||
norm_weight_dtype=norm.weight.dtype,
|
||||
norm_before_gate=bool(norm.norm_before_gate),
|
||||
norm_activation=str(norm.activation),
|
||||
a_log=layer.A_log,
|
||||
dt_bias=layer.dt_bias,
|
||||
state_stride_token=int(ssm_state.stride(0)),
|
||||
@@ -134,6 +143,29 @@ def _qwen_gdn_warmup_config(
|
||||
return None
|
||||
|
||||
|
||||
def _warm_gated_rms_norm_kernel(
|
||||
device: torch.device,
|
||||
config: _QwenGDNWarmupConfig,
|
||||
max_num_tokens: int,
|
||||
x_dtype: torch.dtype,
|
||||
) -> None:
|
||||
from vllm.third_party.flash_linear_attention.ops.layernorm_guard import (
|
||||
warmup_layer_norm_fwd,
|
||||
)
|
||||
|
||||
warmup_layer_norm_fwd(
|
||||
max_num_tokens=max_num_tokens,
|
||||
rows_per_token=config.hv,
|
||||
group_size=config.v,
|
||||
x_dtype=x_dtype,
|
||||
weight_dtype=config.norm_weight_dtype,
|
||||
device=device,
|
||||
norm_before_gate=config.norm_before_gate,
|
||||
is_rms_norm=True,
|
||||
activation=config.norm_activation,
|
||||
)
|
||||
|
||||
|
||||
def _warm_causal_conv1d_fwd_kernel(
|
||||
device: torch.device, config: _QwenGDNWarmupConfig
|
||||
) -> None:
|
||||
@@ -249,7 +281,7 @@ def _synchronize_device(device: torch.device) -> None:
|
||||
@torch.inference_mode()
|
||||
def qwen_triton_warmup(
|
||||
runner: "GPUModelRunner",
|
||||
model_config: object,
|
||||
model_config: "ModelConfig",
|
||||
) -> None:
|
||||
"""Warm Qwen Triton kernels reported by the JIT monitor."""
|
||||
if runner.is_pooling_model:
|
||||
@@ -275,6 +307,8 @@ def qwen_triton_warmup(
|
||||
if gdn_config is None:
|
||||
return
|
||||
|
||||
max_num_tokens = max(1, int(getattr(runner, "max_num_tokens", 1)))
|
||||
_warm_gated_rms_norm_kernel(device, gdn_config, max_num_tokens, model_config.dtype)
|
||||
_warm_causal_conv1d_fwd_kernel(device, gdn_config)
|
||||
_warm_fused_post_conv_kernel(device, gdn_config)
|
||||
_warm_fused_sigmoid_gating_delta_rule_update_kernel(device, gdn_config)
|
||||
|
||||
+270
-25
@@ -13,11 +13,19 @@
|
||||
# This backward pass is faster for dimensions up to 8k, but after that it's much slower due to register spilling.
|
||||
# The models we train have hidden dim up to 8k anyway (e.g. Llama 70B), so this is fine.
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
import torch.nn.functional as F
|
||||
from einops import rearrange
|
||||
|
||||
from vllm.model_executor.warmup.jit_warmup import VllmJitKernel
|
||||
from vllm.model_executor.warmup.jit_warmup_triton_helper import (
|
||||
TritonPointerInputVariant,
|
||||
TritonWarmupTensor,
|
||||
triton_scalar_specialization_rep,
|
||||
)
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.utils.math_utils import cdiv, next_power_of_2
|
||||
from vllm.utils.platform_utils import num_compute_units
|
||||
@@ -175,6 +183,263 @@ def calc_rows_per_block(M: int, device: torch.device) -> int:
|
||||
return rows_per_block
|
||||
|
||||
|
||||
class LayerNormFwdKernel(VllmJitKernel["LayerNormFwdKernel.CompileKey"]):
|
||||
"""Own the runtime and compile-only dispatch for FLA layer norm."""
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class CompileKey:
|
||||
x_dtype: torch.dtype
|
||||
y_dtype: torch.dtype
|
||||
weight_dtype: torch.dtype
|
||||
bias_dtype: torch.dtype | None
|
||||
z_dtype: torch.dtype | None
|
||||
mean_dtype: torch.dtype | None
|
||||
rstd_dtype: torch.dtype
|
||||
input_variant: TritonPointerInputVariant
|
||||
stride_x_row: int
|
||||
stride_y_row: int
|
||||
stride_z_row: int
|
||||
M: int
|
||||
N: int
|
||||
BLOCK_N: int
|
||||
ROWS_PER_BLOCK: int
|
||||
NORM_BEFORE_GATE: bool
|
||||
IS_RMS_NORM: bool
|
||||
ACTIVATION: str
|
||||
|
||||
kernel = staticmethod(layer_norm_fwd_kernel)
|
||||
|
||||
def dispatch( # type: ignore[override]
|
||||
self,
|
||||
*,
|
||||
x_dtype: torch.dtype,
|
||||
y_dtype: torch.dtype,
|
||||
weight_dtype: torch.dtype,
|
||||
bias_dtype: torch.dtype | None,
|
||||
z_dtype: torch.dtype | None,
|
||||
mean_dtype: torch.dtype | None,
|
||||
rstd_dtype: torch.dtype,
|
||||
x_aligned: bool,
|
||||
y_aligned: bool,
|
||||
weight_aligned: bool,
|
||||
bias_aligned: bool,
|
||||
z_aligned: bool,
|
||||
mean_aligned: bool,
|
||||
rstd_aligned: bool,
|
||||
stride_x_row: int,
|
||||
stride_y_row: int,
|
||||
stride_z_row: int,
|
||||
M: int,
|
||||
N: int,
|
||||
BLOCK_N: int,
|
||||
ROWS_PER_BLOCK: int,
|
||||
norm_before_gate: bool,
|
||||
is_rms_norm: bool,
|
||||
activation: str,
|
||||
) -> CompileKey:
|
||||
input_variant = TritonPointerInputVariant.from_alignment(
|
||||
x=x_aligned,
|
||||
y=y_aligned,
|
||||
weight=weight_aligned,
|
||||
bias=bias_aligned,
|
||||
z=z_aligned,
|
||||
mean=mean_aligned,
|
||||
rstd=rstd_aligned,
|
||||
)
|
||||
return self.CompileKey(
|
||||
x_dtype=x_dtype,
|
||||
y_dtype=y_dtype,
|
||||
weight_dtype=weight_dtype,
|
||||
bias_dtype=bias_dtype,
|
||||
z_dtype=z_dtype,
|
||||
mean_dtype=mean_dtype,
|
||||
rstd_dtype=rstd_dtype,
|
||||
input_variant=input_variant,
|
||||
stride_x_row=triton_scalar_specialization_rep(stride_x_row),
|
||||
stride_y_row=triton_scalar_specialization_rep(stride_y_row),
|
||||
stride_z_row=triton_scalar_specialization_rep(stride_z_row),
|
||||
M=triton_scalar_specialization_rep(M),
|
||||
N=N,
|
||||
BLOCK_N=BLOCK_N,
|
||||
ROWS_PER_BLOCK=ROWS_PER_BLOCK,
|
||||
NORM_BEFORE_GATE=norm_before_gate,
|
||||
IS_RMS_NORM=is_rms_norm,
|
||||
ACTIVATION=activation,
|
||||
)
|
||||
|
||||
def get_warmup_keys(
|
||||
self,
|
||||
*,
|
||||
max_num_tokens: int,
|
||||
rows_per_token: int,
|
||||
group_size: int,
|
||||
x_dtype: torch.dtype,
|
||||
weight_dtype: torch.dtype,
|
||||
device: torch.device,
|
||||
norm_before_gate: bool,
|
||||
is_rms_norm: bool,
|
||||
activation: str,
|
||||
) -> list[CompileKey]:
|
||||
if max_num_tokens < 1 or rows_per_token < 1:
|
||||
return []
|
||||
|
||||
max_fused_size = 65536 // torch.empty((), dtype=x_dtype).element_size()
|
||||
block_n = min(max_fused_size, triton.next_power_of_2(group_size))
|
||||
if group_size > block_n:
|
||||
return []
|
||||
|
||||
# Triton's runtime key only distinguishes three integer specialization
|
||||
# classes. Keep one real M for each (M specialization, rows/block) pair.
|
||||
m_values: dict[tuple[int, int], int] = {}
|
||||
for num_tokens in range(1, max_num_tokens + 1):
|
||||
M = num_tokens * rows_per_token
|
||||
signature = (
|
||||
triton_scalar_specialization_rep(M),
|
||||
calc_rows_per_block(M, device),
|
||||
)
|
||||
m_values.setdefault(signature, M)
|
||||
|
||||
keys: dict[LayerNormFwdKernel.CompileKey, None] = {}
|
||||
for M in m_values.values():
|
||||
key = self.dispatch(
|
||||
x_dtype=x_dtype,
|
||||
y_dtype=x_dtype,
|
||||
weight_dtype=weight_dtype,
|
||||
bias_dtype=None,
|
||||
z_dtype=x_dtype,
|
||||
mean_dtype=None if is_rms_norm else torch.float32,
|
||||
rstd_dtype=torch.float32,
|
||||
x_aligned=True,
|
||||
y_aligned=True,
|
||||
weight_aligned=True,
|
||||
bias_aligned=True,
|
||||
z_aligned=True,
|
||||
mean_aligned=True,
|
||||
rstd_aligned=True,
|
||||
stride_x_row=group_size,
|
||||
stride_y_row=group_size,
|
||||
stride_z_row=group_size,
|
||||
M=M,
|
||||
N=group_size,
|
||||
BLOCK_N=block_n,
|
||||
ROWS_PER_BLOCK=calc_rows_per_block(M, device),
|
||||
norm_before_gate=norm_before_gate,
|
||||
is_rms_norm=is_rms_norm,
|
||||
activation=activation,
|
||||
)
|
||||
keys[key] = None
|
||||
return list(keys)
|
||||
|
||||
def compile(self, compile_key: CompileKey) -> None:
|
||||
warmup = getattr(self.kernel, "warmup", None)
|
||||
assert warmup is not None
|
||||
variant = compile_key.input_variant
|
||||
|
||||
def pointer(
|
||||
name: str, dtype: torch.dtype | None
|
||||
) -> TritonWarmupTensor | None:
|
||||
return None if dtype is None else variant.pointer(name, dtype)
|
||||
|
||||
warmup(
|
||||
pointer("x", compile_key.x_dtype),
|
||||
pointer("y", compile_key.y_dtype),
|
||||
pointer("weight", compile_key.weight_dtype),
|
||||
pointer("bias", compile_key.bias_dtype),
|
||||
pointer("z", compile_key.z_dtype),
|
||||
pointer("mean", compile_key.mean_dtype),
|
||||
pointer("rstd", compile_key.rstd_dtype),
|
||||
compile_key.stride_x_row,
|
||||
compile_key.stride_y_row,
|
||||
compile_key.stride_z_row,
|
||||
compile_key.M,
|
||||
compile_key.N,
|
||||
1e-5,
|
||||
BLOCK_N=compile_key.BLOCK_N,
|
||||
ROWS_PER_BLOCK=compile_key.ROWS_PER_BLOCK,
|
||||
HAS_BIAS=compile_key.bias_dtype is not None,
|
||||
HAS_Z=compile_key.z_dtype is not None,
|
||||
NORM_BEFORE_GATE=compile_key.NORM_BEFORE_GATE,
|
||||
IS_RMS_NORM=compile_key.IS_RMS_NORM,
|
||||
ACTIVATION=compile_key.ACTIVATION,
|
||||
num_warps=min(max(compile_key.BLOCK_N // 256, 1), 8),
|
||||
grid=(1, 1),
|
||||
)
|
||||
|
||||
def __call__(
|
||||
self,
|
||||
x: torch.Tensor,
|
||||
out: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
bias: torch.Tensor | None,
|
||||
z: torch.Tensor | None,
|
||||
mean: torch.Tensor | None,
|
||||
rstd: torch.Tensor,
|
||||
eps: float,
|
||||
group_size: int,
|
||||
norm_before_gate: bool,
|
||||
is_rms_norm: bool,
|
||||
activation: str,
|
||||
) -> None:
|
||||
M = x.shape[0]
|
||||
max_fused_size = 65536 // x.element_size()
|
||||
block_n = min(max_fused_size, triton.next_power_of_2(group_size))
|
||||
if group_size > block_n:
|
||||
raise RuntimeError("This layer norm doesn't support feature dim >= 64KB.")
|
||||
rows_per_block = calc_rows_per_block(M, x.device)
|
||||
grid = (cdiv(M, rows_per_block), x.shape[1] // group_size)
|
||||
self.kernel[grid](
|
||||
x,
|
||||
out,
|
||||
weight,
|
||||
bias,
|
||||
z,
|
||||
mean,
|
||||
rstd,
|
||||
x.stride(0),
|
||||
out.stride(0),
|
||||
z.stride(0) if z is not None else 0,
|
||||
M,
|
||||
group_size,
|
||||
eps,
|
||||
BLOCK_N=block_n,
|
||||
ROWS_PER_BLOCK=rows_per_block,
|
||||
HAS_BIAS=bias is not None,
|
||||
HAS_Z=z is not None,
|
||||
NORM_BEFORE_GATE=norm_before_gate,
|
||||
IS_RMS_NORM=is_rms_norm,
|
||||
ACTIVATION=activation,
|
||||
num_warps=min(max(block_n // 256, 1), 8),
|
||||
)
|
||||
|
||||
|
||||
_LAYER_NORM_FWD_KERNEL = LayerNormFwdKernel()
|
||||
|
||||
|
||||
def warmup_layer_norm_fwd(
|
||||
*,
|
||||
max_num_tokens: int,
|
||||
rows_per_token: int,
|
||||
group_size: int,
|
||||
x_dtype: torch.dtype,
|
||||
weight_dtype: torch.dtype,
|
||||
device: torch.device,
|
||||
norm_before_gate: bool,
|
||||
is_rms_norm: bool,
|
||||
activation: str,
|
||||
) -> None:
|
||||
_LAYER_NORM_FWD_KERNEL.warmup(
|
||||
max_num_tokens=max_num_tokens,
|
||||
rows_per_token=rows_per_token,
|
||||
group_size=group_size,
|
||||
x_dtype=x_dtype,
|
||||
weight_dtype=weight_dtype,
|
||||
device=device,
|
||||
norm_before_gate=norm_before_gate,
|
||||
is_rms_norm=is_rms_norm,
|
||||
activation=activation,
|
||||
)
|
||||
|
||||
|
||||
def layer_norm_fwd(
|
||||
x: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
@@ -213,18 +478,7 @@ def layer_norm_fwd(
|
||||
else None
|
||||
)
|
||||
rstd = torch.empty((ngroups * M,), dtype=torch.float32, device=x.device)
|
||||
# Less than 64KB per feature: enqueue fused kernel
|
||||
MAX_FUSED_SIZE = 65536 // x.element_size()
|
||||
BLOCK_N = min(MAX_FUSED_SIZE, triton.next_power_of_2(group_size))
|
||||
if group_size > BLOCK_N:
|
||||
raise RuntimeError("This layer norm doesn't support feature dim >= 64KB.")
|
||||
# heuristics for number of warps
|
||||
num_warps = min(max(BLOCK_N // 256, 1), 8)
|
||||
# Calculate rows per block based on SM count
|
||||
rows_per_block = calc_rows_per_block(M, x.device)
|
||||
# Update grid to use rows_per_block
|
||||
grid = (cdiv(M, rows_per_block), ngroups)
|
||||
layer_norm_fwd_kernel[grid](
|
||||
_LAYER_NORM_FWD_KERNEL(
|
||||
x,
|
||||
out,
|
||||
weight,
|
||||
@@ -232,20 +486,11 @@ def layer_norm_fwd(
|
||||
z,
|
||||
mean,
|
||||
rstd,
|
||||
x.stride(0),
|
||||
out.stride(0),
|
||||
z.stride(0) if z is not None else 0,
|
||||
M,
|
||||
group_size,
|
||||
eps,
|
||||
BLOCK_N=BLOCK_N,
|
||||
ROWS_PER_BLOCK=rows_per_block,
|
||||
HAS_BIAS=bias is not None,
|
||||
HAS_Z=z is not None,
|
||||
NORM_BEFORE_GATE=norm_before_gate,
|
||||
IS_RMS_NORM=is_rms_norm,
|
||||
num_warps=num_warps,
|
||||
ACTIVATION=activation,
|
||||
group_size,
|
||||
norm_before_gate,
|
||||
is_rms_norm,
|
||||
activation,
|
||||
)
|
||||
return out, mean, rstd
|
||||
|
||||
|
||||
Reference in New Issue
Block a user