[Feature] Support batch invariant rms norm with residual (#42453)
Signed-off-by: yewentao256 <zhyanwentao@126.com>
This commit is contained in:
@@ -11,7 +11,9 @@ import pytest
|
||||
import torch
|
||||
from utils import skip_unsupported
|
||||
|
||||
from vllm.model_executor.layers.batch_invariant import rms_norm as triton_rms_norm
|
||||
from vllm.model_executor.layers.batch_invariant import (
|
||||
rms_norm_batch_invariant,
|
||||
)
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
@@ -51,7 +53,7 @@ def test_rms_norm_batch_invariant_vs_standard(
|
||||
standard_output = rms_norm_layer.forward_cuda(input_tensor)
|
||||
|
||||
# Batch-invariant implementation (Triton)
|
||||
triton_output = triton_rms_norm(input_tensor, weight, eps=eps)
|
||||
triton_output = rms_norm_batch_invariant(input_tensor, weight, eps=eps)
|
||||
|
||||
# Compare outputs
|
||||
# Use looser tolerance for bfloat16 due to its lower precision
|
||||
@@ -125,7 +127,7 @@ def test_fused_add_rms_norm_batch_invariant_residual_path(
|
||||
)
|
||||
|
||||
merged_single = x_single + residual_single
|
||||
ref_out = triton_rms_norm(merged_single, weight, eps=eps)
|
||||
ref_out = rms_norm_batch_invariant(merged_single, weight, eps=eps)
|
||||
|
||||
torch.testing.assert_close(
|
||||
residual_out_single,
|
||||
@@ -193,7 +195,7 @@ def test_rms_norm_3d_input(
|
||||
standard_output = rms_norm_layer.forward_cuda(input_tensor)
|
||||
|
||||
# Batch-invariant implementation
|
||||
triton_output = triton_rms_norm(input_tensor, weight, eps=eps)
|
||||
triton_output = rms_norm_batch_invariant(input_tensor, weight, eps=eps)
|
||||
|
||||
# Use looser tolerance for bfloat16
|
||||
rtol, atol = 1e-1, 1e-1 # 10% tolerance for bfloat16
|
||||
@@ -242,7 +244,7 @@ def test_rms_norm_numerical_stability(default_vllm_config):
|
||||
standard_output = rms_norm_layer.forward_cuda(input_tensor)
|
||||
|
||||
# Batch-invariant implementation
|
||||
triton_output = triton_rms_norm(input_tensor, weight, eps=eps)
|
||||
triton_output = rms_norm_batch_invariant(input_tensor, weight, eps=eps)
|
||||
|
||||
# Check for NaN or Inf
|
||||
assert not torch.isnan(standard_output).any(), (
|
||||
@@ -289,7 +291,7 @@ def test_rms_norm_formula(default_vllm_config):
|
||||
expected_output = input_tensor * torch.rsqrt(variance + eps) * weight
|
||||
|
||||
# Batch-invariant implementation
|
||||
triton_output = triton_rms_norm(input_tensor, weight, eps=eps)
|
||||
triton_output = rms_norm_batch_invariant(input_tensor, weight, eps=eps)
|
||||
|
||||
# Compare against formula
|
||||
torch.testing.assert_close(
|
||||
@@ -325,7 +327,7 @@ def test_rms_norm_different_hidden_sizes(default_vllm_config, hidden_size: int):
|
||||
standard_output = rms_norm_layer.forward_cuda(input_tensor)
|
||||
|
||||
# Batch-invariant implementation
|
||||
triton_output = triton_rms_norm(input_tensor, weight, eps=eps)
|
||||
triton_output = rms_norm_batch_invariant(input_tensor, weight, eps=eps)
|
||||
|
||||
# Use looser tolerance for bfloat16
|
||||
rtol, atol = 1e-1, 1e-1 # 10% tolerance for bfloat16
|
||||
@@ -360,7 +362,7 @@ def test_rms_norm_determinism(default_vllm_config):
|
||||
# Run multiple times
|
||||
outputs = []
|
||||
for _ in range(5):
|
||||
output = triton_rms_norm(input_tensor.clone(), weight, eps=eps)
|
||||
output = rms_norm_batch_invariant(input_tensor.clone(), weight, eps=eps)
|
||||
outputs.append(output)
|
||||
|
||||
# All outputs should be identical
|
||||
@@ -395,7 +397,7 @@ if __name__ == "__main__":
|
||||
standard_output = rms_norm_layer.forward_cuda(input_tensor)
|
||||
|
||||
# Batch-invariant implementation
|
||||
triton_output = triton_rms_norm(input_tensor, weight, eps=eps)
|
||||
triton_output = rms_norm_batch_invariant(input_tensor, weight, eps=eps)
|
||||
|
||||
# Compare
|
||||
max_diff = (triton_output - standard_output).abs().max().item()
|
||||
|
||||
@@ -822,23 +822,35 @@ def _rms_norm_kernel(
|
||||
tl.store(output_row_start_ptr + col_idx, output, mask=mask)
|
||||
|
||||
|
||||
def rms_norm(
|
||||
input: torch.Tensor, weight: torch.Tensor, eps: float = 1e-6
|
||||
) -> torch.Tensor:
|
||||
def rms_norm_batch_invariant(
|
||||
input: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
eps: float = 1e-6,
|
||||
residual: torch.Tensor | None = None,
|
||||
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
|
||||
"""
|
||||
Compute RMS normalization using Triton kernel.
|
||||
|
||||
RMS Norm normalizes the input by the root mean square and scales by weight:
|
||||
output = input / sqrt(mean(input^2) + eps) * weight
|
||||
|
||||
Args:
|
||||
input: Input tensor of shape (..., hidden_size)
|
||||
weight: Weight tensor of shape (hidden_size,)
|
||||
eps: Small constant for numerical stability
|
||||
residual: Optional residual tensor fused into the normalization path
|
||||
|
||||
Returns:
|
||||
Tensor with RMS normalization applied along the last dimension
|
||||
RMS normalized tensor, or ``(output, residual_out)`` when ``residual``
|
||||
is provided
|
||||
"""
|
||||
if residual is not None:
|
||||
assert input.shape == residual.shape, (
|
||||
f"Input shape {input.shape} must match residual shape {residual.shape}"
|
||||
)
|
||||
import vllm._custom_ops as ops
|
||||
|
||||
ops.fused_add_rms_norm(input, residual, weight, eps)
|
||||
return input, residual
|
||||
|
||||
assert weight.dim() == 1, "Weight must be 1-dimensional"
|
||||
assert input.shape[-1] == weight.shape[0], (
|
||||
f"Input last dimension ({input.shape[-1]}) must match "
|
||||
@@ -869,26 +881,6 @@ def rms_norm(
|
||||
return output.reshape(original_shape)
|
||||
|
||||
|
||||
def rms_norm_batch_invariant(
|
||||
input: torch.Tensor, weight: torch.Tensor, eps: float = 1e-6
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Batch-invariant wrapper for RMS normalization.
|
||||
|
||||
This function provides a deterministic, batch-invariant implementation
|
||||
of RMS normalization for use with the batch_invariant mode.
|
||||
|
||||
Args:
|
||||
input: Input tensor of shape (..., hidden_size)
|
||||
weight: Weight tensor of shape (hidden_size,)
|
||||
eps: Small constant for numerical stability
|
||||
|
||||
Returns:
|
||||
RMS normalized tensor
|
||||
"""
|
||||
return rms_norm(input, weight, eps=eps)
|
||||
|
||||
|
||||
def linear_batch_invariant(input, weight, bias=None):
|
||||
output = matmul_batch_invariant(input, weight.t())
|
||||
|
||||
|
||||
@@ -106,12 +106,16 @@ class RMSNorm(CustomOp):
|
||||
x: torch.Tensor,
|
||||
residual: torch.Tensor | None = None,
|
||||
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
|
||||
if (
|
||||
envs.VLLM_BATCH_INVARIANT
|
||||
and residual is None
|
||||
and self.variance_size_override is None
|
||||
):
|
||||
return rms_norm_batch_invariant(x, self.weight.data, self.variance_epsilon)
|
||||
if envs.VLLM_BATCH_INVARIANT:
|
||||
assert self.variance_size_override is None, (
|
||||
"Batch invariance is not supported for variance_size_override"
|
||||
)
|
||||
return rms_norm_batch_invariant(
|
||||
x,
|
||||
self.weight.data,
|
||||
self.variance_epsilon,
|
||||
residual=residual,
|
||||
)
|
||||
|
||||
return self.forward_native(x, residual)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user