diff --git a/tests/v1/determinism/test_rms_norm_batch_invariant.py b/tests/v1/determinism/test_rms_norm_batch_invariant.py index 2e9f778812..7fbf8f0461 100644 --- a/tests/v1/determinism/test_rms_norm_batch_invariant.py +++ b/tests/v1/determinism/test_rms_norm_batch_invariant.py @@ -11,7 +11,9 @@ import pytest import torch from utils import skip_unsupported -from vllm.model_executor.layers.batch_invariant import rms_norm as triton_rms_norm +from vllm.model_executor.layers.batch_invariant import ( + rms_norm_batch_invariant, +) from vllm.model_executor.layers.layernorm import RMSNorm from vllm.platforms import current_platform @@ -51,7 +53,7 @@ def test_rms_norm_batch_invariant_vs_standard( standard_output = rms_norm_layer.forward_cuda(input_tensor) # Batch-invariant implementation (Triton) - triton_output = triton_rms_norm(input_tensor, weight, eps=eps) + triton_output = rms_norm_batch_invariant(input_tensor, weight, eps=eps) # Compare outputs # Use looser tolerance for bfloat16 due to its lower precision @@ -125,7 +127,7 @@ def test_fused_add_rms_norm_batch_invariant_residual_path( ) merged_single = x_single + residual_single - ref_out = triton_rms_norm(merged_single, weight, eps=eps) + ref_out = rms_norm_batch_invariant(merged_single, weight, eps=eps) torch.testing.assert_close( residual_out_single, @@ -193,7 +195,7 @@ def test_rms_norm_3d_input( standard_output = rms_norm_layer.forward_cuda(input_tensor) # Batch-invariant implementation - triton_output = triton_rms_norm(input_tensor, weight, eps=eps) + triton_output = rms_norm_batch_invariant(input_tensor, weight, eps=eps) # Use looser tolerance for bfloat16 rtol, atol = 1e-1, 1e-1 # 10% tolerance for bfloat16 @@ -242,7 +244,7 @@ def test_rms_norm_numerical_stability(default_vllm_config): standard_output = rms_norm_layer.forward_cuda(input_tensor) # Batch-invariant implementation - triton_output = triton_rms_norm(input_tensor, weight, eps=eps) + triton_output = rms_norm_batch_invariant(input_tensor, weight, eps=eps) # Check for NaN or Inf assert not torch.isnan(standard_output).any(), ( @@ -289,7 +291,7 @@ def test_rms_norm_formula(default_vllm_config): expected_output = input_tensor * torch.rsqrt(variance + eps) * weight # Batch-invariant implementation - triton_output = triton_rms_norm(input_tensor, weight, eps=eps) + triton_output = rms_norm_batch_invariant(input_tensor, weight, eps=eps) # Compare against formula torch.testing.assert_close( @@ -325,7 +327,7 @@ def test_rms_norm_different_hidden_sizes(default_vllm_config, hidden_size: int): standard_output = rms_norm_layer.forward_cuda(input_tensor) # Batch-invariant implementation - triton_output = triton_rms_norm(input_tensor, weight, eps=eps) + triton_output = rms_norm_batch_invariant(input_tensor, weight, eps=eps) # Use looser tolerance for bfloat16 rtol, atol = 1e-1, 1e-1 # 10% tolerance for bfloat16 @@ -360,7 +362,7 @@ def test_rms_norm_determinism(default_vllm_config): # Run multiple times outputs = [] for _ in range(5): - output = triton_rms_norm(input_tensor.clone(), weight, eps=eps) + output = rms_norm_batch_invariant(input_tensor.clone(), weight, eps=eps) outputs.append(output) # All outputs should be identical @@ -395,7 +397,7 @@ if __name__ == "__main__": standard_output = rms_norm_layer.forward_cuda(input_tensor) # Batch-invariant implementation - triton_output = triton_rms_norm(input_tensor, weight, eps=eps) + triton_output = rms_norm_batch_invariant(input_tensor, weight, eps=eps) # Compare max_diff = (triton_output - standard_output).abs().max().item() diff --git a/vllm/model_executor/layers/batch_invariant.py b/vllm/model_executor/layers/batch_invariant.py index 2e1beeec1b..917c72dee8 100644 --- a/vllm/model_executor/layers/batch_invariant.py +++ b/vllm/model_executor/layers/batch_invariant.py @@ -822,23 +822,35 @@ def _rms_norm_kernel( tl.store(output_row_start_ptr + col_idx, output, mask=mask) -def rms_norm( - input: torch.Tensor, weight: torch.Tensor, eps: float = 1e-6 -) -> torch.Tensor: +def rms_norm_batch_invariant( + input: torch.Tensor, + weight: torch.Tensor, + eps: float = 1e-6, + residual: torch.Tensor | None = None, +) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]: """ Compute RMS normalization using Triton kernel. - RMS Norm normalizes the input by the root mean square and scales by weight: - output = input / sqrt(mean(input^2) + eps) * weight Args: input: Input tensor of shape (..., hidden_size) weight: Weight tensor of shape (hidden_size,) eps: Small constant for numerical stability + residual: Optional residual tensor fused into the normalization path Returns: - Tensor with RMS normalization applied along the last dimension + RMS normalized tensor, or ``(output, residual_out)`` when ``residual`` + is provided """ + if residual is not None: + assert input.shape == residual.shape, ( + f"Input shape {input.shape} must match residual shape {residual.shape}" + ) + import vllm._custom_ops as ops + + ops.fused_add_rms_norm(input, residual, weight, eps) + return input, residual + assert weight.dim() == 1, "Weight must be 1-dimensional" assert input.shape[-1] == weight.shape[0], ( f"Input last dimension ({input.shape[-1]}) must match " @@ -869,26 +881,6 @@ def rms_norm( return output.reshape(original_shape) -def rms_norm_batch_invariant( - input: torch.Tensor, weight: torch.Tensor, eps: float = 1e-6 -) -> torch.Tensor: - """ - Batch-invariant wrapper for RMS normalization. - - This function provides a deterministic, batch-invariant implementation - of RMS normalization for use with the batch_invariant mode. - - Args: - input: Input tensor of shape (..., hidden_size) - weight: Weight tensor of shape (hidden_size,) - eps: Small constant for numerical stability - - Returns: - RMS normalized tensor - """ - return rms_norm(input, weight, eps=eps) - - def linear_batch_invariant(input, weight, bias=None): output = matmul_batch_invariant(input, weight.t()) diff --git a/vllm/model_executor/layers/layernorm.py b/vllm/model_executor/layers/layernorm.py index d5671eb9c1..23027c821d 100644 --- a/vllm/model_executor/layers/layernorm.py +++ b/vllm/model_executor/layers/layernorm.py @@ -106,12 +106,16 @@ class RMSNorm(CustomOp): x: torch.Tensor, residual: torch.Tensor | None = None, ) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]: - if ( - envs.VLLM_BATCH_INVARIANT - and residual is None - and self.variance_size_override is None - ): - return rms_norm_batch_invariant(x, self.weight.data, self.variance_epsilon) + if envs.VLLM_BATCH_INVARIANT: + assert self.variance_size_override is None, ( + "Batch invariance is not supported for variance_size_override" + ) + return rms_norm_batch_invariant( + x, + self.weight.data, + self.variance_epsilon, + residual=residual, + ) return self.forward_native(x, residual)