[Compilation]Fuse Transformers Residual Add + RMSNorm (#48757)
Signed-off-by: BadrBasowid <badr.basowid@gmail.com>
This commit is contained in:
@@ -0,0 +1,105 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
import vllm.ir.ops
|
||||
from tests.compile.backend import TestBackend
|
||||
from vllm.compilation.passes.fusion.add_rms_fusion import (
|
||||
AddRMSNormFusionPass,
|
||||
RMSNormReshapeFusionPass,
|
||||
)
|
||||
from vllm.compilation.passes.fx_utils import find_op_nodes, is_func
|
||||
from vllm.compilation.passes.utility.noop_elimination import NoOpEliminationPass
|
||||
from vllm.compilation.passes.utility.post_cleanup import PostCleanupPass
|
||||
from vllm.config import CompilationConfig, CompilationMode, VllmConfig
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not current_platform.is_cuda_alike(), reason="Requires CUDA or ROCm"
|
||||
)
|
||||
|
||||
|
||||
class RMSNormModel(torch.nn.Module):
|
||||
def __init__(self, hidden_size: int) -> None:
|
||||
super().__init__()
|
||||
self.weight = torch.nn.Parameter(torch.randn(hidden_size))
|
||||
|
||||
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
||||
rms = vllm.ir.ops.rms_norm(x, self.weight, 1e-6)
|
||||
return rms.reshape(-1, rms.shape[-1])
|
||||
|
||||
|
||||
class AddRMSNormModel(torch.nn.Module):
|
||||
def __init__(self, hidden_size: int, residual_first: bool) -> None:
|
||||
super().__init__()
|
||||
self.weight = torch.nn.Parameter(torch.randn(hidden_size))
|
||||
self.residual_first = residual_first
|
||||
|
||||
def forward(
|
||||
self, x: torch.Tensor, residual: torch.Tensor
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
residual = residual + x if self.residual_first else x + residual
|
||||
rms = vllm.ir.ops.rms_norm(residual, self.weight, 1e-6)
|
||||
return rms.reshape(-1, rms.shape[-1]), residual
|
||||
|
||||
def ops_in_model_before(self):
|
||||
return [torch.ops.aten.add, torch.ops.vllm_ir.rms_norm]
|
||||
|
||||
def ops_in_model_after(self):
|
||||
return [torch.ops.vllm_ir.fused_add_rms_norm]
|
||||
|
||||
|
||||
def _run_fusion_test(model, config, passes, *inputs):
|
||||
backend = TestBackend(NoOpEliminationPass(config), *passes, PostCleanupPass(config))
|
||||
outputs_unfused = model(*inputs)
|
||||
outputs_fused = torch.compile(model, backend=backend)(*inputs)
|
||||
torch.testing.assert_close(outputs_fused, outputs_unfused)
|
||||
return backend
|
||||
|
||||
|
||||
def _is_reshape(node):
|
||||
return isinstance(node, torch.fx.Node) and is_func(
|
||||
node, torch.ops.aten.reshape.default
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def vllm_config():
|
||||
config = VllmConfig(
|
||||
compilation_config=CompilationConfig(mode=CompilationMode.VLLM_COMPILE)
|
||||
)
|
||||
with vllm.config.set_current_vllm_config(config):
|
||||
torch.set_default_device("cuda")
|
||||
torch.set_default_dtype(torch.bfloat16)
|
||||
torch.manual_seed(0)
|
||||
yield config
|
||||
|
||||
|
||||
def test_rmsnorm_reshape_fusion(vllm_config):
|
||||
fusion_pass = RMSNormReshapeFusionPass(vllm_config)
|
||||
model = RMSNormModel(hidden_size=32)
|
||||
x = torch.randn(2, 7, 32)
|
||||
backend = _run_fusion_test(model, vllm_config, [fusion_pass], x)
|
||||
|
||||
assert fusion_pass.matched_count == 1
|
||||
(rms_node,) = find_op_nodes(torch.ops.vllm_ir.rms_norm, backend.graph_post_pass)
|
||||
assert _is_reshape(rms_node.args[0])
|
||||
|
||||
|
||||
@pytest.mark.parametrize("residual_first", [True, False])
|
||||
def test_add_rmsnorm_reshape_fusion(vllm_config, residual_first):
|
||||
add_fusion = AddRMSNormFusionPass(vllm_config)
|
||||
reshape_fusion = RMSNormReshapeFusionPass(vllm_config)
|
||||
model = AddRMSNormModel(hidden_size=32, residual_first=residual_first)
|
||||
x = torch.randn(2, 7, 32)
|
||||
residual = torch.randn_like(x)
|
||||
backend = _run_fusion_test(
|
||||
model, vllm_config, [add_fusion, reshape_fusion], x, residual
|
||||
)
|
||||
|
||||
assert add_fusion.matched_count == 1
|
||||
assert reshape_fusion.matched_count == 1
|
||||
backend.check_before_ops(model.ops_in_model_before())
|
||||
backend.check_after_ops(model.ops_in_model_after())
|
||||
@@ -0,0 +1,164 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from itertools import product
|
||||
|
||||
import torch
|
||||
|
||||
import vllm.ir.ops
|
||||
from vllm.config import VllmConfig
|
||||
|
||||
from ..vllm_inductor_pass import (
|
||||
VllmFusionPatternMatcherPass,
|
||||
VllmPatternReplacement,
|
||||
)
|
||||
|
||||
|
||||
class AddRMSNormPattern(VllmPatternReplacement):
|
||||
def __init__(self, epsilon: float, residual_first: bool) -> None:
|
||||
self.epsilon = epsilon
|
||||
self.residual_first = residual_first
|
||||
|
||||
@property
|
||||
def pattern(self):
|
||||
def _pattern(
|
||||
branch: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
residual_out = (
|
||||
residual + branch if self.residual_first else branch + residual
|
||||
)
|
||||
rms = vllm.ir.ops.rms_norm(residual_out, weight, self.epsilon)
|
||||
return rms, residual_out
|
||||
|
||||
return _pattern
|
||||
|
||||
@property
|
||||
def replacement(self):
|
||||
def _replacement(
|
||||
branch: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
return vllm.ir.ops.fused_add_rms_norm(
|
||||
branch, residual, weight, self.epsilon
|
||||
)
|
||||
|
||||
return _replacement
|
||||
|
||||
def get_inputs(self) -> list[torch.Tensor]:
|
||||
return [
|
||||
self.empty_bf16(5, 16), # branch
|
||||
self.empty_bf16(5, 16), # residual
|
||||
self.empty_bf16(16), # weight
|
||||
]
|
||||
|
||||
|
||||
class RMSNormReshapePattern(VllmPatternReplacement):
|
||||
"""Move a prefix-flatten before RMSNorm."""
|
||||
|
||||
def __init__(self, epsilon: float) -> None:
|
||||
self.epsilon = epsilon
|
||||
|
||||
@property
|
||||
def pattern(self):
|
||||
def _pattern(
|
||||
input: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
rms = vllm.ir.ops.rms_norm(input, weight, self.epsilon)
|
||||
return rms.reshape(-1, rms.shape[-1])
|
||||
|
||||
return _pattern
|
||||
|
||||
@property
|
||||
def replacement(self):
|
||||
def _replacement(
|
||||
input: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
input = input.reshape(-1, input.shape[-1])
|
||||
return vllm.ir.ops.rms_norm(input, weight, self.epsilon)
|
||||
|
||||
return _replacement
|
||||
|
||||
def get_inputs(self) -> list[torch.Tensor]:
|
||||
return [
|
||||
self.empty_bf16(1, 5, 16), # input
|
||||
self.empty_bf16(16), # weight
|
||||
]
|
||||
|
||||
|
||||
class FusedAddRMSNormReshapePattern(VllmPatternReplacement):
|
||||
"""Move a prefix-flatten before fused add RMSNorm."""
|
||||
|
||||
def __init__(self, epsilon: float) -> None:
|
||||
self.epsilon = epsilon
|
||||
|
||||
@property
|
||||
def pattern(self):
|
||||
def _pattern(
|
||||
input: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
rms, residual_out = vllm.ir.ops.fused_add_rms_norm(
|
||||
input, residual, weight, self.epsilon
|
||||
)
|
||||
return rms.reshape(-1, rms.shape[-1]), residual_out
|
||||
|
||||
return _pattern
|
||||
|
||||
@property
|
||||
def replacement(self):
|
||||
def _replacement(
|
||||
input: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
original_shape = input.shape
|
||||
hidden_size = input.shape[-1]
|
||||
rms, residual_out = vllm.ir.ops.fused_add_rms_norm(
|
||||
input.reshape(-1, hidden_size),
|
||||
residual.reshape(-1, hidden_size),
|
||||
weight,
|
||||
self.epsilon,
|
||||
)
|
||||
return rms, residual_out.reshape(original_shape)
|
||||
|
||||
return _replacement
|
||||
|
||||
def get_inputs(self) -> list[torch.Tensor]:
|
||||
return [
|
||||
self.empty_bf16(1, 5, 16), # input
|
||||
self.empty_bf16(1, 5, 16), # residual
|
||||
self.empty_bf16(16), # weight
|
||||
]
|
||||
|
||||
|
||||
class AddRMSNormFusionPass(VllmFusionPatternMatcherPass):
|
||||
"""Fuse residual Add and RMSNorm emitted separately by the Transformers backend."""
|
||||
|
||||
def __init__(self, config: VllmConfig) -> None:
|
||||
super().__init__(config, "add_rmsnorm_fusion_pass")
|
||||
|
||||
for epsilon, residual_first in product([1e-5, 1e-6], [True, False]):
|
||||
self.register(AddRMSNormPattern(epsilon, residual_first))
|
||||
|
||||
self.dump_patterns(config, self.pm_pass)
|
||||
|
||||
|
||||
class RMSNormReshapeFusionPass(VllmFusionPatternMatcherPass):
|
||||
"""Move the Transformers backend's post-RMSNorm flatten before the norm
|
||||
for downstream 2D fusions.
|
||||
"""
|
||||
|
||||
def __init__(self, config: VllmConfig) -> None:
|
||||
super().__init__(config, "rmsnorm_reshape_fusion_pass")
|
||||
|
||||
for epsilon in [1e-5, 1e-6]:
|
||||
self.register(FusedAddRMSNormReshapePattern(epsilon))
|
||||
self.register(RMSNormReshapePattern(epsilon))
|
||||
|
||||
self.dump_patterns(config, self.pm_pass)
|
||||
@@ -30,6 +30,10 @@ if rocm_aiter_ops.is_enabled():
|
||||
)
|
||||
|
||||
if current_platform.is_cuda_alike() or current_platform.is_xpu():
|
||||
from .fusion.add_rms_fusion import (
|
||||
AddRMSNormFusionPass,
|
||||
RMSNormReshapeFusionPass,
|
||||
)
|
||||
from .fusion.qk_norm_rope_fusion import QKNormRoPEFusionPass
|
||||
from .fusion.sequence_parallelism import SequenceParallelismPass
|
||||
from .utility.split_coalescing import SplitCoalescingPass
|
||||
@@ -138,6 +142,16 @@ class PostGradPassManager(CustomGraphPass): # type: ignore[misc]
|
||||
|
||||
def configure(self, config: VllmConfig) -> None:
|
||||
self.pass_config = config.compilation_config.pass_config
|
||||
model_config = config.model_config
|
||||
enable_transformers_norm_canonicalization = (
|
||||
(
|
||||
self.pass_config.fuse_act_padding
|
||||
or self.pass_config.fuse_allreduce_rms
|
||||
or self.pass_config.fuse_norm_quant
|
||||
)
|
||||
and model_config is not None
|
||||
and model_config.using_transformers_backend()
|
||||
)
|
||||
|
||||
# Set the current vllm config to allow tracing CustomOp instances
|
||||
with set_current_vllm_config(config, check_compile=False):
|
||||
@@ -149,6 +163,9 @@ class PostGradPassManager(CustomGraphPass): # type: ignore[misc]
|
||||
if self.pass_config.fuse_gemm_comms:
|
||||
self.passes += [AsyncTPPass(config)]
|
||||
|
||||
if enable_transformers_norm_canonicalization:
|
||||
self.passes += [AddRMSNormFusionPass(config)]
|
||||
|
||||
if self.pass_config.fuse_act_padding and rocm_aiter_ops.is_enabled():
|
||||
# Run the more specific RMSNorm+router-pad fusion before
|
||||
# AR+RMS, since both consume fused_add_rms_norm.
|
||||
@@ -160,6 +177,11 @@ class PostGradPassManager(CustomGraphPass): # type: ignore[misc]
|
||||
else:
|
||||
self.passes += [AllReduceFusionPass(config)]
|
||||
|
||||
if enable_transformers_norm_canonicalization:
|
||||
# Let AR+RMS match before moving output reshapes ahead of the
|
||||
# remaining RMSNorms, exposing them to RMS+Quant fusion.
|
||||
self.passes += [RMSNormReshapeFusionPass(config)]
|
||||
|
||||
if self.pass_config.fuse_norm_quant:
|
||||
if rocm_aiter_ops.is_enabled():
|
||||
self.passes += [
|
||||
|
||||
@@ -75,7 +75,6 @@ def _rms_norm_fake(x: Tensor, weight: Tensor, variance_epsilon: float) -> Tensor
|
||||
direct_register_aiter_op(
|
||||
op_name="rms_norm", op_func=_rms_norm_impl, fake_impl=_rms_norm_fake
|
||||
)
|
||||
|
||||
rms_add_no_var_16bit_only = (
|
||||
lambda x, x_residual, weight, epsilon, variance_size=None: variance_size is None
|
||||
and x.dtype in (torch.float16, torch.bfloat16)
|
||||
@@ -112,6 +111,12 @@ def _rocm_aiter_rmsnorm2d_fwd_with_add_impl(
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
from aiter import rmsnorm2d_fwd_with_add
|
||||
|
||||
x_shape = x.shape
|
||||
residual_shape = residual.shape
|
||||
hidden_size = x.shape[-1]
|
||||
x = x.reshape(-1, hidden_size)
|
||||
residual = residual.reshape(-1, hidden_size)
|
||||
|
||||
# TODO can out = x and residual_out = residual to save memory?
|
||||
# Need to check if the kernel supports in-place residual output
|
||||
# (if yes set mutates_args and inplace)
|
||||
@@ -125,7 +130,7 @@ def _rocm_aiter_rmsnorm2d_fwd_with_add_impl(
|
||||
weight,
|
||||
variance_epsilon,
|
||||
)
|
||||
return out, residual_out
|
||||
return out.reshape(x_shape), residual_out.reshape(residual_shape)
|
||||
|
||||
|
||||
def _rocm_aiter_rmsnorm2d_fwd_with_add_fake(
|
||||
|
||||
Reference in New Issue
Block a user