[Feature] Support batch invariant rms norm with residual (#42453)

Signed-off-by: yewentao256 <zhyanwentao@126.com>
This commit is contained in:
Wentao Ye
2026-06-03 15:22:01 -04:00
committed by GitHub
parent a248b45d05
commit dad95e34d8
3 changed files with 39 additions and 41 deletions
+18 -26
View File
@@ -822,23 +822,35 @@ def _rms_norm_kernel(
tl.store(output_row_start_ptr + col_idx, output, mask=mask)
def rms_norm(
input: torch.Tensor, weight: torch.Tensor, eps: float = 1e-6
) -> torch.Tensor:
def rms_norm_batch_invariant(
input: torch.Tensor,
weight: torch.Tensor,
eps: float = 1e-6,
residual: torch.Tensor | None = None,
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
"""
Compute RMS normalization using Triton kernel.
RMS Norm normalizes the input by the root mean square and scales by weight:
output = input / sqrt(mean(input^2) + eps) * weight
Args:
input: Input tensor of shape (..., hidden_size)
weight: Weight tensor of shape (hidden_size,)
eps: Small constant for numerical stability
residual: Optional residual tensor fused into the normalization path
Returns:
Tensor with RMS normalization applied along the last dimension
RMS normalized tensor, or ``(output, residual_out)`` when ``residual``
is provided
"""
if residual is not None:
assert input.shape == residual.shape, (
f"Input shape {input.shape} must match residual shape {residual.shape}"
)
import vllm._custom_ops as ops
ops.fused_add_rms_norm(input, residual, weight, eps)
return input, residual
assert weight.dim() == 1, "Weight must be 1-dimensional"
assert input.shape[-1] == weight.shape[0], (
f"Input last dimension ({input.shape[-1]}) must match "
@@ -869,26 +881,6 @@ def rms_norm(
return output.reshape(original_shape)
def rms_norm_batch_invariant(
input: torch.Tensor, weight: torch.Tensor, eps: float = 1e-6
) -> torch.Tensor:
"""
Batch-invariant wrapper for RMS normalization.
This function provides a deterministic, batch-invariant implementation
of RMS normalization for use with the batch_invariant mode.
Args:
input: Input tensor of shape (..., hidden_size)
weight: Weight tensor of shape (hidden_size,)
eps: Small constant for numerical stability
Returns:
RMS normalized tensor
"""
return rms_norm(input, weight, eps=eps)
def linear_batch_invariant(input, weight, bias=None):
output = matmul_batch_invariant(input, weight.t())
+10 -6
View File
@@ -106,12 +106,16 @@ class RMSNorm(CustomOp):
x: torch.Tensor,
residual: torch.Tensor | None = None,
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
if (
envs.VLLM_BATCH_INVARIANT
and residual is None
and self.variance_size_override is None
):
return rms_norm_batch_invariant(x, self.weight.data, self.variance_epsilon)
if envs.VLLM_BATCH_INVARIANT:
assert self.variance_size_override is None, (
"Batch invariance is not supported for variance_size_override"
)
return rms_norm_batch_invariant(
x,
self.weight.data,
self.variance_epsilon,
residual=residual,
)
return self.forward_native(x, residual)