[Kernel] Add weightless RMSNorm CUDA kernels for has_weight=False (#41430) (#44109)

Signed-off-by: hello-args <args.sarkar@gmail.com>
This commit is contained in:
arghyadeep sarkar
2026-06-16 23:45:55 -07:00
committed by GitHub
parent 17bc144556
commit 93bbe94d3a
11 changed files with 231 additions and 117 deletions
+48
View File
@@ -6,6 +6,7 @@ import torch
from tests.kernels.quant_utils import FP8_DTYPE
from tests.kernels.utils import opcheck
from vllm import ir
from vllm.model_executor.layers.layernorm import GemmaRMSNorm, RMSNorm
from vllm.platforms import current_platform
from vllm.utils.torch_utils import set_random_seed
@@ -27,6 +28,10 @@ CUDA_DEVICES = [
]
def _rms_norm_tolerance(dtype: torch.dtype) -> dict[str, float]:
return ir.ops.rms_norm.get_tolerance(dtype)
@pytest.mark.parametrize("num_tokens", NUM_TOKENS)
@pytest.mark.parametrize("hidden_size", HIDDEN_SIZES)
@pytest.mark.parametrize("add_residual", ADD_RESIDUAL)
@@ -81,6 +86,49 @@ def test_rms_norm(
)
@pytest.mark.parametrize("num_tokens", NUM_TOKENS)
@pytest.mark.parametrize("hidden_size", HIDDEN_SIZES)
@pytest.mark.parametrize("add_residual", ADD_RESIDUAL)
@pytest.mark.parametrize("dtype", DTYPES)
@pytest.mark.parametrize("seed", SEEDS)
@pytest.mark.parametrize("device", CUDA_DEVICES)
@torch.inference_mode()
def test_rms_norm_weightless(
default_vllm_config,
num_tokens: int,
hidden_size: int,
add_residual: bool,
dtype: torch.dtype,
seed: int,
device: str,
) -> None:
set_random_seed(seed)
torch.set_default_device(device)
layer = RMSNorm(hidden_size, has_weight=False).to(dtype=dtype)
x = torch.randn(num_tokens, hidden_size, dtype=dtype)
residual = torch.randn_like(x) if add_residual else None
ref_out = layer.forward_native(x, residual)
out = layer(x, residual)
tol = _rms_norm_tolerance(dtype)
if add_residual:
torch.testing.assert_close(out[0], ref_out[0], **tol)
torch.testing.assert_close(out[1], ref_out[1], **tol)
else:
torch.testing.assert_close(out, ref_out, **tol)
if residual is not None:
opcheck(
torch.ops._C.fused_add_rms_norm,
(x, residual, None, layer.variance_epsilon),
)
else:
opcheck(
torch.ops._C.rms_norm,
(out, x, None, layer.variance_epsilon),
)
@pytest.mark.parametrize("num_tokens", NUM_TOKENS)
@pytest.mark.parametrize("hidden_size", HIDDEN_SIZES)
@pytest.mark.parametrize("add_residual", ADD_RESIDUAL)