[Feature] Support batch invariant rms norm with residual (#42453)

Signed-off-by: yewentao256 <zhyanwentao@126.com>
This commit is contained in:
Wentao Ye
2026-06-03 15:22:01 -04:00
committed by GitHub
parent a248b45d05
commit dad95e34d8
3 changed files with 39 additions and 41 deletions
@@ -11,7 +11,9 @@ import pytest
import torch
from utils import skip_unsupported
from vllm.model_executor.layers.batch_invariant import rms_norm as triton_rms_norm
from vllm.model_executor.layers.batch_invariant import (
rms_norm_batch_invariant,
)
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.platforms import current_platform
@@ -51,7 +53,7 @@ def test_rms_norm_batch_invariant_vs_standard(
standard_output = rms_norm_layer.forward_cuda(input_tensor)
# Batch-invariant implementation (Triton)
triton_output = triton_rms_norm(input_tensor, weight, eps=eps)
triton_output = rms_norm_batch_invariant(input_tensor, weight, eps=eps)
# Compare outputs
# Use looser tolerance for bfloat16 due to its lower precision
@@ -125,7 +127,7 @@ def test_fused_add_rms_norm_batch_invariant_residual_path(
)
merged_single = x_single + residual_single
ref_out = triton_rms_norm(merged_single, weight, eps=eps)
ref_out = rms_norm_batch_invariant(merged_single, weight, eps=eps)
torch.testing.assert_close(
residual_out_single,
@@ -193,7 +195,7 @@ def test_rms_norm_3d_input(
standard_output = rms_norm_layer.forward_cuda(input_tensor)
# Batch-invariant implementation
triton_output = triton_rms_norm(input_tensor, weight, eps=eps)
triton_output = rms_norm_batch_invariant(input_tensor, weight, eps=eps)
# Use looser tolerance for bfloat16
rtol, atol = 1e-1, 1e-1 # 10% tolerance for bfloat16
@@ -242,7 +244,7 @@ def test_rms_norm_numerical_stability(default_vllm_config):
standard_output = rms_norm_layer.forward_cuda(input_tensor)
# Batch-invariant implementation
triton_output = triton_rms_norm(input_tensor, weight, eps=eps)
triton_output = rms_norm_batch_invariant(input_tensor, weight, eps=eps)
# Check for NaN or Inf
assert not torch.isnan(standard_output).any(), (
@@ -289,7 +291,7 @@ def test_rms_norm_formula(default_vllm_config):
expected_output = input_tensor * torch.rsqrt(variance + eps) * weight
# Batch-invariant implementation
triton_output = triton_rms_norm(input_tensor, weight, eps=eps)
triton_output = rms_norm_batch_invariant(input_tensor, weight, eps=eps)
# Compare against formula
torch.testing.assert_close(
@@ -325,7 +327,7 @@ def test_rms_norm_different_hidden_sizes(default_vllm_config, hidden_size: int):
standard_output = rms_norm_layer.forward_cuda(input_tensor)
# Batch-invariant implementation
triton_output = triton_rms_norm(input_tensor, weight, eps=eps)
triton_output = rms_norm_batch_invariant(input_tensor, weight, eps=eps)
# Use looser tolerance for bfloat16
rtol, atol = 1e-1, 1e-1 # 10% tolerance for bfloat16
@@ -360,7 +362,7 @@ def test_rms_norm_determinism(default_vllm_config):
# Run multiple times
outputs = []
for _ in range(5):
output = triton_rms_norm(input_tensor.clone(), weight, eps=eps)
output = rms_norm_batch_invariant(input_tensor.clone(), weight, eps=eps)
outputs.append(output)
# All outputs should be identical
@@ -395,7 +397,7 @@ if __name__ == "__main__":
standard_output = rms_norm_layer.forward_cuda(input_tensor)
# Batch-invariant implementation
triton_output = triton_rms_norm(input_tensor, weight, eps=eps)
triton_output = rms_norm_batch_invariant(input_tensor, weight, eps=eps)
# Compare
max_diff = (triton_output - standard_output).abs().max().item()
+18 -26
View File
@@ -822,23 +822,35 @@ def _rms_norm_kernel(
tl.store(output_row_start_ptr + col_idx, output, mask=mask)
def rms_norm(
input: torch.Tensor, weight: torch.Tensor, eps: float = 1e-6
) -> torch.Tensor:
def rms_norm_batch_invariant(
input: torch.Tensor,
weight: torch.Tensor,
eps: float = 1e-6,
residual: torch.Tensor | None = None,
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
"""
Compute RMS normalization using Triton kernel.
RMS Norm normalizes the input by the root mean square and scales by weight:
output = input / sqrt(mean(input^2) + eps) * weight
Args:
input: Input tensor of shape (..., hidden_size)
weight: Weight tensor of shape (hidden_size,)
eps: Small constant for numerical stability
residual: Optional residual tensor fused into the normalization path
Returns:
Tensor with RMS normalization applied along the last dimension
RMS normalized tensor, or ``(output, residual_out)`` when ``residual``
is provided
"""
if residual is not None:
assert input.shape == residual.shape, (
f"Input shape {input.shape} must match residual shape {residual.shape}"
)
import vllm._custom_ops as ops
ops.fused_add_rms_norm(input, residual, weight, eps)
return input, residual
assert weight.dim() == 1, "Weight must be 1-dimensional"
assert input.shape[-1] == weight.shape[0], (
f"Input last dimension ({input.shape[-1]}) must match "
@@ -869,26 +881,6 @@ def rms_norm(
return output.reshape(original_shape)
def rms_norm_batch_invariant(
input: torch.Tensor, weight: torch.Tensor, eps: float = 1e-6
) -> torch.Tensor:
"""
Batch-invariant wrapper for RMS normalization.
This function provides a deterministic, batch-invariant implementation
of RMS normalization for use with the batch_invariant mode.
Args:
input: Input tensor of shape (..., hidden_size)
weight: Weight tensor of shape (hidden_size,)
eps: Small constant for numerical stability
Returns:
RMS normalized tensor
"""
return rms_norm(input, weight, eps=eps)
def linear_batch_invariant(input, weight, bias=None):
output = matmul_batch_invariant(input, weight.t())
+10 -6
View File
@@ -106,12 +106,16 @@ class RMSNorm(CustomOp):
x: torch.Tensor,
residual: torch.Tensor | None = None,
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
if (
envs.VLLM_BATCH_INVARIANT
and residual is None
and self.variance_size_override is None
):
return rms_norm_batch_invariant(x, self.weight.data, self.variance_epsilon)
if envs.VLLM_BATCH_INVARIANT:
assert self.variance_size_override is None, (
"Batch invariance is not supported for variance_size_override"
)
return rms_norm_batch_invariant(
x,
self.weight.data,
self.variance_epsilon,
residual=residual,
)
return self.forward_native(x, residual)