From 59e831c09a22bdd2732b86a158ff771584b14793 Mon Sep 17 00:00:00 2001 From: BadrBasowid <61441185+BadrBasowid@users.noreply.github.com> Date: Thu, 30 Jul 2026 21:46:36 +0800 Subject: [PATCH] [Compilation]Fuse Transformers Residual Add + RMSNorm (#48757) Signed-off-by: BadrBasowid --- .../passes/test_rmsnorm_reshape_fusion.py | 105 +++++++++++ .../passes/fusion/add_rms_fusion.py | 164 ++++++++++++++++++ vllm/compilation/passes/pass_manager.py | 22 +++ vllm/kernels/aiter_ops.py | 9 +- 4 files changed, 298 insertions(+), 2 deletions(-) create mode 100644 tests/compile/passes/test_rmsnorm_reshape_fusion.py create mode 100644 vllm/compilation/passes/fusion/add_rms_fusion.py diff --git a/tests/compile/passes/test_rmsnorm_reshape_fusion.py b/tests/compile/passes/test_rmsnorm_reshape_fusion.py new file mode 100644 index 0000000000..410b2e97eb --- /dev/null +++ b/tests/compile/passes/test_rmsnorm_reshape_fusion.py @@ -0,0 +1,105 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import pytest +import torch + +import vllm.ir.ops +from tests.compile.backend import TestBackend +from vllm.compilation.passes.fusion.add_rms_fusion import ( + AddRMSNormFusionPass, + RMSNormReshapeFusionPass, +) +from vllm.compilation.passes.fx_utils import find_op_nodes, is_func +from vllm.compilation.passes.utility.noop_elimination import NoOpEliminationPass +from vllm.compilation.passes.utility.post_cleanup import PostCleanupPass +from vllm.config import CompilationConfig, CompilationMode, VllmConfig +from vllm.platforms import current_platform + +pytestmark = pytest.mark.skipif( + not current_platform.is_cuda_alike(), reason="Requires CUDA or ROCm" +) + + +class RMSNormModel(torch.nn.Module): + def __init__(self, hidden_size: int) -> None: + super().__init__() + self.weight = torch.nn.Parameter(torch.randn(hidden_size)) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + rms = vllm.ir.ops.rms_norm(x, self.weight, 1e-6) + return rms.reshape(-1, rms.shape[-1]) + + +class AddRMSNormModel(torch.nn.Module): + def __init__(self, hidden_size: int, residual_first: bool) -> None: + super().__init__() + self.weight = torch.nn.Parameter(torch.randn(hidden_size)) + self.residual_first = residual_first + + def forward( + self, x: torch.Tensor, residual: torch.Tensor + ) -> tuple[torch.Tensor, torch.Tensor]: + residual = residual + x if self.residual_first else x + residual + rms = vllm.ir.ops.rms_norm(residual, self.weight, 1e-6) + return rms.reshape(-1, rms.shape[-1]), residual + + def ops_in_model_before(self): + return [torch.ops.aten.add, torch.ops.vllm_ir.rms_norm] + + def ops_in_model_after(self): + return [torch.ops.vllm_ir.fused_add_rms_norm] + + +def _run_fusion_test(model, config, passes, *inputs): + backend = TestBackend(NoOpEliminationPass(config), *passes, PostCleanupPass(config)) + outputs_unfused = model(*inputs) + outputs_fused = torch.compile(model, backend=backend)(*inputs) + torch.testing.assert_close(outputs_fused, outputs_unfused) + return backend + + +def _is_reshape(node): + return isinstance(node, torch.fx.Node) and is_func( + node, torch.ops.aten.reshape.default + ) + + +@pytest.fixture +def vllm_config(): + config = VllmConfig( + compilation_config=CompilationConfig(mode=CompilationMode.VLLM_COMPILE) + ) + with vllm.config.set_current_vllm_config(config): + torch.set_default_device("cuda") + torch.set_default_dtype(torch.bfloat16) + torch.manual_seed(0) + yield config + + +def test_rmsnorm_reshape_fusion(vllm_config): + fusion_pass = RMSNormReshapeFusionPass(vllm_config) + model = RMSNormModel(hidden_size=32) + x = torch.randn(2, 7, 32) + backend = _run_fusion_test(model, vllm_config, [fusion_pass], x) + + assert fusion_pass.matched_count == 1 + (rms_node,) = find_op_nodes(torch.ops.vllm_ir.rms_norm, backend.graph_post_pass) + assert _is_reshape(rms_node.args[0]) + + +@pytest.mark.parametrize("residual_first", [True, False]) +def test_add_rmsnorm_reshape_fusion(vllm_config, residual_first): + add_fusion = AddRMSNormFusionPass(vllm_config) + reshape_fusion = RMSNormReshapeFusionPass(vllm_config) + model = AddRMSNormModel(hidden_size=32, residual_first=residual_first) + x = torch.randn(2, 7, 32) + residual = torch.randn_like(x) + backend = _run_fusion_test( + model, vllm_config, [add_fusion, reshape_fusion], x, residual + ) + + assert add_fusion.matched_count == 1 + assert reshape_fusion.matched_count == 1 + backend.check_before_ops(model.ops_in_model_before()) + backend.check_after_ops(model.ops_in_model_after()) diff --git a/vllm/compilation/passes/fusion/add_rms_fusion.py b/vllm/compilation/passes/fusion/add_rms_fusion.py new file mode 100644 index 0000000000..b61c2b36ad --- /dev/null +++ b/vllm/compilation/passes/fusion/add_rms_fusion.py @@ -0,0 +1,164 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from itertools import product + +import torch + +import vllm.ir.ops +from vllm.config import VllmConfig + +from ..vllm_inductor_pass import ( + VllmFusionPatternMatcherPass, + VllmPatternReplacement, +) + + +class AddRMSNormPattern(VllmPatternReplacement): + def __init__(self, epsilon: float, residual_first: bool) -> None: + self.epsilon = epsilon + self.residual_first = residual_first + + @property + def pattern(self): + def _pattern( + branch: torch.Tensor, + residual: torch.Tensor, + weight: torch.Tensor, + ) -> tuple[torch.Tensor, torch.Tensor]: + residual_out = ( + residual + branch if self.residual_first else branch + residual + ) + rms = vllm.ir.ops.rms_norm(residual_out, weight, self.epsilon) + return rms, residual_out + + return _pattern + + @property + def replacement(self): + def _replacement( + branch: torch.Tensor, + residual: torch.Tensor, + weight: torch.Tensor, + ) -> tuple[torch.Tensor, torch.Tensor]: + return vllm.ir.ops.fused_add_rms_norm( + branch, residual, weight, self.epsilon + ) + + return _replacement + + def get_inputs(self) -> list[torch.Tensor]: + return [ + self.empty_bf16(5, 16), # branch + self.empty_bf16(5, 16), # residual + self.empty_bf16(16), # weight + ] + + +class RMSNormReshapePattern(VllmPatternReplacement): + """Move a prefix-flatten before RMSNorm.""" + + def __init__(self, epsilon: float) -> None: + self.epsilon = epsilon + + @property + def pattern(self): + def _pattern( + input: torch.Tensor, + weight: torch.Tensor, + ) -> torch.Tensor: + rms = vllm.ir.ops.rms_norm(input, weight, self.epsilon) + return rms.reshape(-1, rms.shape[-1]) + + return _pattern + + @property + def replacement(self): + def _replacement( + input: torch.Tensor, + weight: torch.Tensor, + ) -> torch.Tensor: + input = input.reshape(-1, input.shape[-1]) + return vllm.ir.ops.rms_norm(input, weight, self.epsilon) + + return _replacement + + def get_inputs(self) -> list[torch.Tensor]: + return [ + self.empty_bf16(1, 5, 16), # input + self.empty_bf16(16), # weight + ] + + +class FusedAddRMSNormReshapePattern(VllmPatternReplacement): + """Move a prefix-flatten before fused add RMSNorm.""" + + def __init__(self, epsilon: float) -> None: + self.epsilon = epsilon + + @property + def pattern(self): + def _pattern( + input: torch.Tensor, + residual: torch.Tensor, + weight: torch.Tensor, + ) -> tuple[torch.Tensor, torch.Tensor]: + rms, residual_out = vllm.ir.ops.fused_add_rms_norm( + input, residual, weight, self.epsilon + ) + return rms.reshape(-1, rms.shape[-1]), residual_out + + return _pattern + + @property + def replacement(self): + def _replacement( + input: torch.Tensor, + residual: torch.Tensor, + weight: torch.Tensor, + ) -> tuple[torch.Tensor, torch.Tensor]: + original_shape = input.shape + hidden_size = input.shape[-1] + rms, residual_out = vllm.ir.ops.fused_add_rms_norm( + input.reshape(-1, hidden_size), + residual.reshape(-1, hidden_size), + weight, + self.epsilon, + ) + return rms, residual_out.reshape(original_shape) + + return _replacement + + def get_inputs(self) -> list[torch.Tensor]: + return [ + self.empty_bf16(1, 5, 16), # input + self.empty_bf16(1, 5, 16), # residual + self.empty_bf16(16), # weight + ] + + +class AddRMSNormFusionPass(VllmFusionPatternMatcherPass): + """Fuse residual Add and RMSNorm emitted separately by the Transformers backend.""" + + def __init__(self, config: VllmConfig) -> None: + super().__init__(config, "add_rmsnorm_fusion_pass") + + for epsilon, residual_first in product([1e-5, 1e-6], [True, False]): + self.register(AddRMSNormPattern(epsilon, residual_first)) + + self.dump_patterns(config, self.pm_pass) + + +class RMSNormReshapeFusionPass(VllmFusionPatternMatcherPass): + """Move the Transformers backend's post-RMSNorm flatten before the norm + for downstream 2D fusions. + """ + + def __init__(self, config: VllmConfig) -> None: + super().__init__(config, "rmsnorm_reshape_fusion_pass") + + for epsilon in [1e-5, 1e-6]: + self.register(FusedAddRMSNormReshapePattern(epsilon)) + self.register(RMSNormReshapePattern(epsilon)) + + self.dump_patterns(config, self.pm_pass) diff --git a/vllm/compilation/passes/pass_manager.py b/vllm/compilation/passes/pass_manager.py index 127be115c5..acc9004498 100644 --- a/vllm/compilation/passes/pass_manager.py +++ b/vllm/compilation/passes/pass_manager.py @@ -30,6 +30,10 @@ if rocm_aiter_ops.is_enabled(): ) if current_platform.is_cuda_alike() or current_platform.is_xpu(): + from .fusion.add_rms_fusion import ( + AddRMSNormFusionPass, + RMSNormReshapeFusionPass, + ) from .fusion.qk_norm_rope_fusion import QKNormRoPEFusionPass from .fusion.sequence_parallelism import SequenceParallelismPass from .utility.split_coalescing import SplitCoalescingPass @@ -138,6 +142,16 @@ class PostGradPassManager(CustomGraphPass): # type: ignore[misc] def configure(self, config: VllmConfig) -> None: self.pass_config = config.compilation_config.pass_config + model_config = config.model_config + enable_transformers_norm_canonicalization = ( + ( + self.pass_config.fuse_act_padding + or self.pass_config.fuse_allreduce_rms + or self.pass_config.fuse_norm_quant + ) + and model_config is not None + and model_config.using_transformers_backend() + ) # Set the current vllm config to allow tracing CustomOp instances with set_current_vllm_config(config, check_compile=False): @@ -149,6 +163,9 @@ class PostGradPassManager(CustomGraphPass): # type: ignore[misc] if self.pass_config.fuse_gemm_comms: self.passes += [AsyncTPPass(config)] + if enable_transformers_norm_canonicalization: + self.passes += [AddRMSNormFusionPass(config)] + if self.pass_config.fuse_act_padding and rocm_aiter_ops.is_enabled(): # Run the more specific RMSNorm+router-pad fusion before # AR+RMS, since both consume fused_add_rms_norm. @@ -160,6 +177,11 @@ class PostGradPassManager(CustomGraphPass): # type: ignore[misc] else: self.passes += [AllReduceFusionPass(config)] + if enable_transformers_norm_canonicalization: + # Let AR+RMS match before moving output reshapes ahead of the + # remaining RMSNorms, exposing them to RMS+Quant fusion. + self.passes += [RMSNormReshapeFusionPass(config)] + if self.pass_config.fuse_norm_quant: if rocm_aiter_ops.is_enabled(): self.passes += [ diff --git a/vllm/kernels/aiter_ops.py b/vllm/kernels/aiter_ops.py index 273bc58935..de8a53ae8b 100644 --- a/vllm/kernels/aiter_ops.py +++ b/vllm/kernels/aiter_ops.py @@ -75,7 +75,6 @@ def _rms_norm_fake(x: Tensor, weight: Tensor, variance_epsilon: float) -> Tensor direct_register_aiter_op( op_name="rms_norm", op_func=_rms_norm_impl, fake_impl=_rms_norm_fake ) - rms_add_no_var_16bit_only = ( lambda x, x_residual, weight, epsilon, variance_size=None: variance_size is None and x.dtype in (torch.float16, torch.bfloat16) @@ -112,6 +111,12 @@ def _rocm_aiter_rmsnorm2d_fwd_with_add_impl( ) -> tuple[torch.Tensor, torch.Tensor]: from aiter import rmsnorm2d_fwd_with_add + x_shape = x.shape + residual_shape = residual.shape + hidden_size = x.shape[-1] + x = x.reshape(-1, hidden_size) + residual = residual.reshape(-1, hidden_size) + # TODO can out = x and residual_out = residual to save memory? # Need to check if the kernel supports in-place residual output # (if yes set mutates_args and inplace) @@ -125,7 +130,7 @@ def _rocm_aiter_rmsnorm2d_fwd_with_add_impl( weight, variance_epsilon, ) - return out, residual_out + return out.reshape(x_shape), residual_out.reshape(residual_shape) def _rocm_aiter_rmsnorm2d_fwd_with_add_fake(