[Compilation]Fuse Transformers Residual Add + RMSNorm (#48757)

Signed-off-by: BadrBasowid <badr.basowid@gmail.com>
This commit is contained in:
BadrBasowid
2026-07-30 13:46:36 +00:00
committed by GitHub
parent 1a20d23dab
commit 59e831c09a
4 changed files with 298 additions and 2 deletions
@@ -0,0 +1,105 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import torch
import vllm.ir.ops
from tests.compile.backend import TestBackend
from vllm.compilation.passes.fusion.add_rms_fusion import (
AddRMSNormFusionPass,
RMSNormReshapeFusionPass,
)
from vllm.compilation.passes.fx_utils import find_op_nodes, is_func
from vllm.compilation.passes.utility.noop_elimination import NoOpEliminationPass
from vllm.compilation.passes.utility.post_cleanup import PostCleanupPass
from vllm.config import CompilationConfig, CompilationMode, VllmConfig
from vllm.platforms import current_platform
pytestmark = pytest.mark.skipif(
not current_platform.is_cuda_alike(), reason="Requires CUDA or ROCm"
)
class RMSNormModel(torch.nn.Module):
def __init__(self, hidden_size: int) -> None:
super().__init__()
self.weight = torch.nn.Parameter(torch.randn(hidden_size))
def forward(self, x: torch.Tensor) -> torch.Tensor:
rms = vllm.ir.ops.rms_norm(x, self.weight, 1e-6)
return rms.reshape(-1, rms.shape[-1])
class AddRMSNormModel(torch.nn.Module):
def __init__(self, hidden_size: int, residual_first: bool) -> None:
super().__init__()
self.weight = torch.nn.Parameter(torch.randn(hidden_size))
self.residual_first = residual_first
def forward(
self, x: torch.Tensor, residual: torch.Tensor
) -> tuple[torch.Tensor, torch.Tensor]:
residual = residual + x if self.residual_first else x + residual
rms = vllm.ir.ops.rms_norm(residual, self.weight, 1e-6)
return rms.reshape(-1, rms.shape[-1]), residual
def ops_in_model_before(self):
return [torch.ops.aten.add, torch.ops.vllm_ir.rms_norm]
def ops_in_model_after(self):
return [torch.ops.vllm_ir.fused_add_rms_norm]
def _run_fusion_test(model, config, passes, *inputs):
backend = TestBackend(NoOpEliminationPass(config), *passes, PostCleanupPass(config))
outputs_unfused = model(*inputs)
outputs_fused = torch.compile(model, backend=backend)(*inputs)
torch.testing.assert_close(outputs_fused, outputs_unfused)
return backend
def _is_reshape(node):
return isinstance(node, torch.fx.Node) and is_func(
node, torch.ops.aten.reshape.default
)
@pytest.fixture
def vllm_config():
config = VllmConfig(
compilation_config=CompilationConfig(mode=CompilationMode.VLLM_COMPILE)
)
with vllm.config.set_current_vllm_config(config):
torch.set_default_device("cuda")
torch.set_default_dtype(torch.bfloat16)
torch.manual_seed(0)
yield config
def test_rmsnorm_reshape_fusion(vllm_config):
fusion_pass = RMSNormReshapeFusionPass(vllm_config)
model = RMSNormModel(hidden_size=32)
x = torch.randn(2, 7, 32)
backend = _run_fusion_test(model, vllm_config, [fusion_pass], x)
assert fusion_pass.matched_count == 1
(rms_node,) = find_op_nodes(torch.ops.vllm_ir.rms_norm, backend.graph_post_pass)
assert _is_reshape(rms_node.args[0])
@pytest.mark.parametrize("residual_first", [True, False])
def test_add_rmsnorm_reshape_fusion(vllm_config, residual_first):
add_fusion = AddRMSNormFusionPass(vllm_config)
reshape_fusion = RMSNormReshapeFusionPass(vllm_config)
model = AddRMSNormModel(hidden_size=32, residual_first=residual_first)
x = torch.randn(2, 7, 32)
residual = torch.randn_like(x)
backend = _run_fusion_test(
model, vllm_config, [add_fusion, reshape_fusion], x, residual
)
assert add_fusion.matched_count == 1
assert reshape_fusion.matched_count == 1
backend.check_before_ops(model.ops_in_model_before())
backend.check_after_ops(model.ops_in_model_after())
@@ -0,0 +1,164 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from itertools import product
import torch
import vllm.ir.ops
from vllm.config import VllmConfig
from ..vllm_inductor_pass import (
VllmFusionPatternMatcherPass,
VllmPatternReplacement,
)
class AddRMSNormPattern(VllmPatternReplacement):
def __init__(self, epsilon: float, residual_first: bool) -> None:
self.epsilon = epsilon
self.residual_first = residual_first
@property
def pattern(self):
def _pattern(
branch: torch.Tensor,
residual: torch.Tensor,
weight: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor]:
residual_out = (
residual + branch if self.residual_first else branch + residual
)
rms = vllm.ir.ops.rms_norm(residual_out, weight, self.epsilon)
return rms, residual_out
return _pattern
@property
def replacement(self):
def _replacement(
branch: torch.Tensor,
residual: torch.Tensor,
weight: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor]:
return vllm.ir.ops.fused_add_rms_norm(
branch, residual, weight, self.epsilon
)
return _replacement
def get_inputs(self) -> list[torch.Tensor]:
return [
self.empty_bf16(5, 16), # branch
self.empty_bf16(5, 16), # residual
self.empty_bf16(16), # weight
]
class RMSNormReshapePattern(VllmPatternReplacement):
"""Move a prefix-flatten before RMSNorm."""
def __init__(self, epsilon: float) -> None:
self.epsilon = epsilon
@property
def pattern(self):
def _pattern(
input: torch.Tensor,
weight: torch.Tensor,
) -> torch.Tensor:
rms = vllm.ir.ops.rms_norm(input, weight, self.epsilon)
return rms.reshape(-1, rms.shape[-1])
return _pattern
@property
def replacement(self):
def _replacement(
input: torch.Tensor,
weight: torch.Tensor,
) -> torch.Tensor:
input = input.reshape(-1, input.shape[-1])
return vllm.ir.ops.rms_norm(input, weight, self.epsilon)
return _replacement
def get_inputs(self) -> list[torch.Tensor]:
return [
self.empty_bf16(1, 5, 16), # input
self.empty_bf16(16), # weight
]
class FusedAddRMSNormReshapePattern(VllmPatternReplacement):
"""Move a prefix-flatten before fused add RMSNorm."""
def __init__(self, epsilon: float) -> None:
self.epsilon = epsilon
@property
def pattern(self):
def _pattern(
input: torch.Tensor,
residual: torch.Tensor,
weight: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor]:
rms, residual_out = vllm.ir.ops.fused_add_rms_norm(
input, residual, weight, self.epsilon
)
return rms.reshape(-1, rms.shape[-1]), residual_out
return _pattern
@property
def replacement(self):
def _replacement(
input: torch.Tensor,
residual: torch.Tensor,
weight: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor]:
original_shape = input.shape
hidden_size = input.shape[-1]
rms, residual_out = vllm.ir.ops.fused_add_rms_norm(
input.reshape(-1, hidden_size),
residual.reshape(-1, hidden_size),
weight,
self.epsilon,
)
return rms, residual_out.reshape(original_shape)
return _replacement
def get_inputs(self) -> list[torch.Tensor]:
return [
self.empty_bf16(1, 5, 16), # input
self.empty_bf16(1, 5, 16), # residual
self.empty_bf16(16), # weight
]
class AddRMSNormFusionPass(VllmFusionPatternMatcherPass):
"""Fuse residual Add and RMSNorm emitted separately by the Transformers backend."""
def __init__(self, config: VllmConfig) -> None:
super().__init__(config, "add_rmsnorm_fusion_pass")
for epsilon, residual_first in product([1e-5, 1e-6], [True, False]):
self.register(AddRMSNormPattern(epsilon, residual_first))
self.dump_patterns(config, self.pm_pass)
class RMSNormReshapeFusionPass(VllmFusionPatternMatcherPass):
"""Move the Transformers backend's post-RMSNorm flatten before the norm
for downstream 2D fusions.
"""
def __init__(self, config: VllmConfig) -> None:
super().__init__(config, "rmsnorm_reshape_fusion_pass")
for epsilon in [1e-5, 1e-6]:
self.register(FusedAddRMSNormReshapePattern(epsilon))
self.register(RMSNormReshapePattern(epsilon))
self.dump_patterns(config, self.pm_pass)
+22
View File
@@ -30,6 +30,10 @@ if rocm_aiter_ops.is_enabled():
)
if current_platform.is_cuda_alike() or current_platform.is_xpu():
from .fusion.add_rms_fusion import (
AddRMSNormFusionPass,
RMSNormReshapeFusionPass,
)
from .fusion.qk_norm_rope_fusion import QKNormRoPEFusionPass
from .fusion.sequence_parallelism import SequenceParallelismPass
from .utility.split_coalescing import SplitCoalescingPass
@@ -138,6 +142,16 @@ class PostGradPassManager(CustomGraphPass): # type: ignore[misc]
def configure(self, config: VllmConfig) -> None:
self.pass_config = config.compilation_config.pass_config
model_config = config.model_config
enable_transformers_norm_canonicalization = (
(
self.pass_config.fuse_act_padding
or self.pass_config.fuse_allreduce_rms
or self.pass_config.fuse_norm_quant
)
and model_config is not None
and model_config.using_transformers_backend()
)
# Set the current vllm config to allow tracing CustomOp instances
with set_current_vllm_config(config, check_compile=False):
@@ -149,6 +163,9 @@ class PostGradPassManager(CustomGraphPass): # type: ignore[misc]
if self.pass_config.fuse_gemm_comms:
self.passes += [AsyncTPPass(config)]
if enable_transformers_norm_canonicalization:
self.passes += [AddRMSNormFusionPass(config)]
if self.pass_config.fuse_act_padding and rocm_aiter_ops.is_enabled():
# Run the more specific RMSNorm+router-pad fusion before
# AR+RMS, since both consume fused_add_rms_norm.
@@ -160,6 +177,11 @@ class PostGradPassManager(CustomGraphPass): # type: ignore[misc]
else:
self.passes += [AllReduceFusionPass(config)]
if enable_transformers_norm_canonicalization:
# Let AR+RMS match before moving output reshapes ahead of the
# remaining RMSNorms, exposing them to RMS+Quant fusion.
self.passes += [RMSNormReshapeFusionPass(config)]
if self.pass_config.fuse_norm_quant:
if rocm_aiter_ops.is_enabled():
self.passes += [
+7 -2
View File
@@ -75,7 +75,6 @@ def _rms_norm_fake(x: Tensor, weight: Tensor, variance_epsilon: float) -> Tensor
direct_register_aiter_op(
op_name="rms_norm", op_func=_rms_norm_impl, fake_impl=_rms_norm_fake
)
rms_add_no_var_16bit_only = (
lambda x, x_residual, weight, epsilon, variance_size=None: variance_size is None
and x.dtype in (torch.float16, torch.bfloat16)
@@ -112,6 +111,12 @@ def _rocm_aiter_rmsnorm2d_fwd_with_add_impl(
) -> tuple[torch.Tensor, torch.Tensor]:
from aiter import rmsnorm2d_fwd_with_add
x_shape = x.shape
residual_shape = residual.shape
hidden_size = x.shape[-1]
x = x.reshape(-1, hidden_size)
residual = residual.reshape(-1, hidden_size)
# TODO can out = x and residual_out = residual to save memory?
# Need to check if the kernel supports in-place residual output
# (if yes set mutates_args and inplace)
@@ -125,7 +130,7 @@ def _rocm_aiter_rmsnorm2d_fwd_with_add_impl(
weight,
variance_epsilon,
)
return out, residual_out
return out.reshape(x_shape), residual_out.reshape(residual_shape)
def _rocm_aiter_rmsnorm2d_fwd_with_add_fake(