diff --git a/vllm/ir/ops/layernorm.py b/vllm/ir/ops/layernorm.py index ac0c38a9e4..8471aa043c 100644 --- a/vllm/ir/ops/layernorm.py +++ b/vllm/ir/ops/layernorm.py @@ -16,6 +16,7 @@ def rms_norm( x_var = x if variance_size is None else x[..., :variance_size] variance = x_var.pow(2).mean(dim=-1, keepdim=True) x = x * torch.rsqrt(variance + epsilon) + x = x.to(orig_dtype) if weight is not None: - x = x.to(weight.dtype) * weight - return x.to(orig_dtype) + x = x * weight + return x diff --git a/vllm/kernels/aiter_ops.py b/vllm/kernels/aiter_ops.py index 14c2b87fbb..1980051dd9 100644 --- a/vllm/kernels/aiter_ops.py +++ b/vllm/kernels/aiter_ops.py @@ -36,11 +36,13 @@ AITER_SUPPORTED = is_aiter_found() rms_no_var_16bit_only = ( lambda x, weight, epsilon, variance_size=None: variance_size is None - and x.dtype in (torch.float16, torch.bfloat16) - and (weight is None or weight.dtype == x.dtype) + and x.dtype + in ( + torch.float16, + torch.bfloat16, + ) ) -"""AITER rms_norm only supports float16 and bfloat16 acts, no var_size override, -and requires weight dtype to match x dtype.""" +"""AITER rms_norm only supports float16 and bfloat16 acts and no var_size override.""" @ir.ops.rms_norm.register_impl( diff --git a/vllm/kernels/vllm_c.py b/vllm/kernels/vllm_c.py index fab91de2ea..fabb36d7b4 100644 --- a/vllm/kernels/vllm_c.py +++ b/vllm/kernels/vllm_c.py @@ -11,11 +11,8 @@ current_platform.import_kernels() CUDA_ALIKE = current_platform.is_cuda_alike() """Most kernels in this file are supported on all CUDA-alike platforms.""" -rms_no_var_size = ( - lambda x, weight, epsilon, variance_size=None: variance_size is None - and (weight is None or weight.dtype == x.dtype) -) -"""vLLM kernel does not support variance_size parameter or mismatched weight dtype.""" +rms_no_var_size = lambda x, weight, epsilon, variance_size=None: variance_size is None +"""vLLM kernel does not support variance_size parameter.""" @ir.ops.rms_norm.register_impl( diff --git a/vllm/kernels/xpu_ops.py b/vllm/kernels/xpu_ops.py index c680c542c1..3548fb8682 100644 --- a/vllm/kernels/xpu_ops.py +++ b/vllm/kernels/xpu_ops.py @@ -18,9 +18,7 @@ def is_xpu_kernels_found() -> bool: XPU_KERNELS_SUPPORTED = is_xpu_kernels_found() """Kernels in this file are supported if vLLM XPU kernels are installed.""" -rms_no_var = lambda x, weight, epsilon, variance_size=None: variance_size is None and ( - weight is None or weight.dtype == x.dtype -) +rms_no_var = lambda x, weight, epsilon, variance_size=None: variance_size is None @ir.ops.rms_norm.register_impl( diff --git a/vllm/model_executor/layers/layernorm.py b/vllm/model_executor/layers/layernorm.py index df22f3102b..7b222f9c43 100644 --- a/vllm/model_executor/layers/layernorm.py +++ b/vllm/model_executor/layers/layernorm.py @@ -376,6 +376,46 @@ class GemmaRMSNorm(CustomOp): self.weight = nn.Parameter(torch.zeros(hidden_size)) self.variance_epsilon = eps + @staticmethod + def _forward_static_no_residual( + weight: torch.Tensor, + variance_epsilon: float, + x: torch.Tensor, + ) -> torch.Tensor: + """PyTorch-native implementation equivalent to forward() without residual.""" + orig_dtype = x.dtype + x = x.float() + variance = x.pow(2).mean(dim=-1, keepdim=True) + x = x * torch.rsqrt(variance + variance_epsilon) + x = x * (1.0 + weight.float()) + x = x.to(orig_dtype) + return x + + @staticmethod + def _forward_static_with_residual( + weight: torch.Tensor, + variance_epsilon: float, + x: torch.Tensor, + residual: torch.Tensor, + ) -> tuple[torch.Tensor, torch.Tensor]: + """PyTorch-native implementation equivalent to forward() with residual.""" + orig_dtype = x.dtype + x = ( + x.float() + residual.float() + if orig_dtype == torch.float16 + else x + residual + ) + residual = x + + x = x.float() + variance = x.pow(2).mean(dim=-1, keepdim=True) + x = x * torch.rsqrt(variance + variance_epsilon) + # Llama does x.to(float16) * w whilst Gemma is (x * w).to(float16) + # See https://github.com/huggingface/transformers/pull/29402 + x = x * (1.0 + weight.float()) + x = x.to(orig_dtype) + return x, residual + def forward_native( self, x: torch.Tensor, @@ -383,26 +423,30 @@ class GemmaRMSNorm(CustomOp): ) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]: """PyTorch-native implementation equivalent to forward().""" if residual is None: - return ir.ops.rms_norm( - x, self.weight.data.float() + 1.0, self.variance_epsilon + return self._forward_static_no_residual( + self.weight.data, self.variance_epsilon, x ) else: - orig_dtype = x.dtype - x = ( - x.float() + residual.float() - if orig_dtype == torch.float16 - else x + residual + return self._forward_static_with_residual( + self.weight.data, self.variance_epsilon, x, residual ) - residual = x - return ir.ops.rms_norm( - x, self.weight.data.float() + 1.0, self.variance_epsilon - ).to(orig_dtype), residual def forward_cuda( self, x: torch.Tensor, residual: torch.Tensor | None = None, ) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]: + if torch.compiler.is_compiling(): + return self.forward_native(x, residual) + + if not getattr(self, "_is_compiled", False): + self._forward_static_no_residual = torch.compile( # type: ignore + self._forward_static_no_residual + ) + self._forward_static_with_residual = torch.compile( # type: ignore + self._forward_static_with_residual + ) + self._is_compiled = True return self.forward_native(x, residual)