[vLLM IR] 2/N fused_add_rms_norm and maybe_inplace overload (#36823)

Signed-off-by: Luka Govedič <lgovedic@redhat.com>
Signed-off-by: Luka Govedič <ProExpertProg@users.noreply.github.com>
This commit is contained in:
Luka Govedič
2026-05-01 23:41:15 -04:00
committed by GitHub
parent 3e49479c4b
commit d58c42e19c
44 changed files with 2833 additions and 779 deletions
+15 -2
View File
@@ -12,10 +12,17 @@ from torch._ops import OpOverload, OpOverloadPacket
from torch.fx._utils import lazy_format_graph_code
from vllm.compilation.passes.fx_utils import find_op_nodes
from vllm.compilation.passes.inductor_pass import InductorPass
from vllm.compilation.passes.inductor_pass import (
InductorPass,
pass_context,
)
from vllm.compilation.passes.ir.inplace_functionalization import (
VllmIRInplaceFunctionalizationPass,
)
from vllm.compilation.passes.pass_manager import with_pattern_match_debug
from vllm.compilation.passes.vllm_inductor_pass import VllmInductorPass
from vllm.config import VllmConfig, get_current_vllm_config
from vllm.config.utils import Range
from vllm.logger import init_logger
logger = init_logger("vllm.tests.compile.backend")
@@ -53,11 +60,17 @@ class TestBackend:
self.custom_passes = list(passes)
vllm_config = get_current_vllm_config()
compile_config = vllm_config.compilation_config
self.range = Range(1, vllm_config.scheduler_config.max_num_batched_tokens)
# Deepcopy to allow multiple TestBackend instances to use the same VllmConfig
self.inductor_config = deepcopy(compile_config.inductor_compile_config)
self.inductor_config["force_disable_caches"] = True
self.inductor_config["post_grad_custom_post_pass"] = self.post_pass
# Add VllmIRInplaceFunctionalizationPass as pre-grad pass by default
self.inductor_config["pre_grad_custom_pass"] = (
VllmIRInplaceFunctionalizationPass(vllm_config)
)
if debug_dump_path := vllm_config.compile_debug_dump_path():
logger.debug("Dumping depyf output to %s", debug_dump_path)
self.debug_ctx = depyf.prepare_debug(debug_dump_path.as_posix())
@@ -68,7 +81,7 @@ class TestBackend:
self.graph_pre_compile = deepcopy(graph)
from torch._inductor.compile_fx import compile_fx
with self.debug_ctx:
with self.debug_ctx, pass_context(self.range):
return compile_fx(
graph, example_inputs, config_patches=self.inductor_config
)
@@ -88,14 +88,10 @@ class TestAllReduceRMSNormModel(torch.nn.Module):
]
def ops_in_model(self):
return (
[torch.ops.vllm_ir.rms_norm]
+ [
torch.ops._C.fused_add_rms_norm.default,
]
if RMSNorm.enabled()
else []
)
return [
torch.ops.vllm_ir.rms_norm,
torch.ops.vllm_ir.fused_add_rms_norm,
]
class TestAllReduceRMSNormStaticQuantFP8Model(torch.nn.Module):
@@ -152,16 +148,17 @@ class TestAllReduceRMSNormStaticQuantFP8Model(torch.nn.Module):
def ops_in_model(self):
if self.vllm_config.compilation_config.pass_config.fuse_norm_quant:
return [torch.ops._C.fused_add_rms_norm_static_fp8_quant.default]
elif RMSNorm.enabled():
return [
torch.ops._C.fused_add_rms_norm.default,
]
elif any(layer.is_quant_fp8_enabled() for layer in self.fp8_linear_layers):
return [
torch.ops._C.static_scaled_fp8_quant.default,
]
else:
return []
quant_ops = (
[torch.ops._C.static_scaled_fp8_quant.default]
if any(layer.is_quant_fp8_enabled() for layer in self.fp8_linear_layers)
else [torch.ops.aten.reciprocal]
)
return [
torch.ops.vllm_ir.rms_norm,
torch.ops.vllm_ir.fused_add_rms_norm,
*quant_ops,
]
@multi_gpu_test(num_gpus=2)
@@ -0,0 +1,412 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Comprehensive tests for UnsafeCloneEliminationPass.
This test suite exercises all possible valid FX graph patterns involving clones:
1. Clone with no users (dead code)
2. Clone with read-only users
3. Clone with mutation users
4. Clone of graph input
5. Clone with original used after mutation
6. Clone chains
"""
import pytest
import torch
from torch import fx
from torch.fx.experimental.proxy_tensor import make_fx
from vllm.compilation.passes.fx_utils import find_op_nodes
from vllm.compilation.passes.inductor_pass import get_pass_context, pass_context
from vllm.compilation.passes.ir.clone_elimination import (
UnsafeCloneEliminationPass,
user_writes_to_node,
)
from vllm.config import VllmConfig
from vllm.config.utils import Range
def count_clones(graph: fx.Graph) -> int:
"""Count clone nodes in a graph."""
return len(list(find_op_nodes(torch.ops.aten.clone.default, graph)))
@pytest.fixture(scope="function")
def clone_cleanup_pass():
return UnsafeCloneEliminationPass(VllmConfig())
@pytest.fixture(autouse=True)
def setup_pass_context():
"""Set up pass context for each test."""
with pass_context(compile_range=Range(1, 8192)):
yield
class TestCloneCleanup:
"""Test UnsafeCloneEliminationPass behavior on various graph patterns."""
def test_remove_clone_readonly_users(self, clone_cleanup_pass):
"""Clone with only read-only users should be removed."""
def f(x: torch.Tensor) -> torch.Tensor:
x_clone = x.clone()
return x_clone + 1
inp = torch.randn(2, 3)
graph_module = make_fx(f)(inp)
assert count_clones(graph_module.graph) == 1
expected = graph_module(inp)
clone_cleanup_pass(graph_module.graph)
graph_module.recompile()
actual = graph_module(inp)
assert count_clones(graph_module.graph) == 0
torch.testing.assert_close(actual, expected)
def test_keep_clone_with_mutation_and_original_used_after(self, clone_cleanup_pass):
"""Clone must be kept if it's mutated AND original is used after mutation."""
def f(x: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
x = x.relu() # not a graph param
x_clone = x.clone()
x_clone.add_(1)
return x, x_clone
inp = torch.randn(2, 3)
graph_module = make_fx(f)(inp)
assert count_clones(graph_module.graph) == 1
expected = graph_module(inp)
clone_cleanup_pass(graph_module.graph)
graph_module.recompile()
actual = graph_module(inp)
# Clone should be KEPT because original is used after mutation
assert count_clones(graph_module.graph) == 1
torch.testing.assert_close(actual[0], expected[0])
torch.testing.assert_close(actual[1], expected[1])
def test_remove_clone_with_mutation_no_original_use(self, clone_cleanup_pass):
"""Clone can be removed if it's mutated but original is not used after."""
def f(x: torch.Tensor) -> torch.Tensor:
x = x.relu() # not a graph param
x_clone = x.clone()
x_clone.add_(1)
return x_clone
inp = torch.randn(2, 3)
graph_module = make_fx(f)(inp)
assert count_clones(graph_module.graph) == 1
expected = graph_module(inp)
clone_cleanup_pass(graph_module.graph)
graph_module.recompile()
actual = graph_module(inp)
assert count_clones(graph_module.graph) == 0
torch.testing.assert_close(actual, expected)
def test_clone_chain(self, clone_cleanup_pass):
"""Test handling of clone chains: x -> clone1 -> clone2."""
def f(x: torch.Tensor) -> torch.Tensor:
x = x.relu() # not a graph param
x1 = x.clone()
x2 = x1.clone()
return x2 + 1
inp = torch.randn(2, 3)
graph_module = make_fx(f)(inp)
assert count_clones(graph_module.graph) == 2
expected = graph_module(inp)
clone_cleanup_pass(graph_module.graph)
graph_module.recompile()
actual = graph_module(inp)
# Both clones should be removed
assert count_clones(graph_module.graph) == 0
torch.testing.assert_close(actual, expected)
def test_multiple_clones_of_same_input(self, clone_cleanup_pass):
"""Test multiple independent clones of the same input."""
def f(x: torch.Tensor) -> torch.Tensor:
x1 = x.clone()
x2 = x.clone()
return x1 + x2
inp = torch.randn(2, 3)
graph_module = make_fx(f)(inp)
assert count_clones(graph_module.graph) == 2
expected = graph_module(inp)
clone_cleanup_pass(graph_module.graph)
graph_module.recompile()
actual = graph_module(inp)
# Both clones should be removed (only readonly uses)
assert count_clones(graph_module.graph) == 0
torch.testing.assert_close(actual, expected)
def test_no_clones_in_graph(self, clone_cleanup_pass):
"""Test pass behavior when graph has no clones."""
def f(x: torch.Tensor) -> torch.Tensor:
return x + 1
inp = torch.randn(2, 3)
graph_module = make_fx(f)(inp)
assert count_clones(graph_module.graph) == 0
expected = graph_module(inp)
clone_cleanup_pass(graph_module.graph)
graph_module.recompile()
actual = graph_module(inp)
assert count_clones(graph_module.graph) == 0
torch.testing.assert_close(actual, expected)
def test_multiple_passes(self, clone_cleanup_pass):
"""Test running the pass multiple times (should be idempotent)."""
def f(x: torch.Tensor) -> torch.Tensor:
x1 = x.clone()
return x1 + 1
inp = torch.randn(2, 3)
graph_module = make_fx(f)(inp)
assert count_clones(graph_module.graph) == 1
expected = graph_module(inp)
clone_cleanup_pass(graph_module.graph)
assert count_clones(graph_module.graph) == 0
graph_module.recompile()
actual = graph_module(inp)
torch.testing.assert_close(actual, expected)
clone_cleanup_pass(graph_module.graph)
assert count_clones(graph_module.graph) == 0
graph_module.recompile()
actual = graph_module(inp)
torch.testing.assert_close(actual, expected)
def test_output_node_no_write(self):
"""Output nodes never write to their inputs."""
def f(x: torch.Tensor) -> torch.Tensor:
return x
graph_module = make_fx(f)(torch.randn(2, 3))
x_node = [n for n in graph_module.graph.nodes if n.op == "placeholder"][0]
output_node = [n for n in graph_module.graph.nodes if n.op == "output"][0]
assert not user_writes_to_node(output_node, x_node)
def test_readonly_op_no_write(self):
"""Readonly operations don't write to inputs."""
def f(x: torch.Tensor, y: torch.Tensor) -> torch.Tensor:
return x + y
graph_module = make_fx(f)(torch.randn(2, 3), torch.randn(2, 3))
placeholders = [n for n in graph_module.graph.nodes if n.op == "placeholder"]
add_node = [
n
for n in graph_module.graph.nodes
if n.op == "call_function" and n.target == torch.ops.aten.add.Tensor
][0]
assert not user_writes_to_node(add_node, placeholders[0])
assert not user_writes_to_node(add_node, placeholders[1])
def test_inplace_op_writes(self):
"""Inplace operations write to first argument."""
def f(x: torch.Tensor, y: torch.Tensor) -> torch.Tensor:
x.add_(y)
return x
graph_module = make_fx(f)(torch.randn(2, 3), torch.randn(2, 3))
placeholders = [n for n in graph_module.graph.nodes if n.op == "placeholder"]
add_node = [
n
for n in graph_module.graph.nodes
if n.op == "call_function" and "add_" in str(n.target)
][0]
# add_ writes to first arg but not second
assert user_writes_to_node(add_node, placeholders[0])
assert not user_writes_to_node(add_node, placeholders[1])
def test_copy_writes(self):
"""copy_ operation writes to first argument."""
def f(x: torch.Tensor, y: torch.Tensor) -> torch.Tensor:
x.copy_(y)
return x
graph_module = make_fx(f)(torch.randn(2, 3), torch.randn(2, 3))
placeholders = [n for n in graph_module.graph.nodes if n.op == "placeholder"]
copy_node = [
n
for n in graph_module.graph.nodes
if n.op == "call_function" and "copy_" in str(n.target)
][0]
assert user_writes_to_node(copy_node, placeholders[0])
assert not user_writes_to_node(copy_node, placeholders[1])
def test_auto_functionalized_not_a_write(self):
"""auto_functionalized ops are follow-up uses, not writes."""
from torch._higher_order_ops.auto_functionalize import auto_functionalized
def f(x: torch.Tensor) -> torch.Tensor:
return x
graph_module = make_fx(f)(torch.randn(2, 3))
x_node = [n for n in graph_module.graph.nodes if n.op == "placeholder"][0]
# Create an auto_functionalized node in the graph
with graph_module.graph.inserting_before(None):
af_node = graph_module.graph.call_function(
auto_functionalized, kwargs={"input": x_node}
)
# auto_functionalized should not be treated as a write
assert not user_writes_to_node(af_node, x_node)
def test_higher_order_op_conservatively_writes(self):
"""Other higher-order operators are conservatively treated as writes."""
from torch._ops import HigherOrderOperator
def f(x: torch.Tensor) -> torch.Tensor:
return x
graph_module = make_fx(f)(torch.randn(2, 3))
x_node = [n for n in graph_module.graph.nodes if n.op == "placeholder"][0]
# Create a concrete higher-order operator subclass
class MockHigherOrderOp(HigherOrderOperator):
def __call__(self, *args, **kwargs):
return args[0] if args else None
mock_hoo = MockHigherOrderOp("mock_higher_order_op")
with graph_module.graph.inserting_before(None):
hoo_node = graph_module.graph.call_function(mock_hoo, args=(x_node,))
# Should be conservative and assume it could write
assert user_writes_to_node(hoo_node, x_node)
class TestCloneCleanupWithDonatedInputs:
"""Test UnsafeCloneEliminationPass with donated input tracking via PassContext."""
@pytest.fixture(autouse=True)
def setup_pass_context(self):
"""Set up pass context for each test."""
with pass_context(compile_range=Range(1, 8192)):
yield
def test_donated_input_clone_removed(self, clone_cleanup_pass):
"""Clone of donated input should be removed."""
def f(x: torch.Tensor) -> torch.Tensor:
x_clone = x.clone()
x_clone.add_(1)
return x_clone
inp = torch.randn(2, 3)
graph_module = make_fx(f)(inp)
assert count_clones(graph_module.graph) == 1
# Mark first parameter as donated
get_pass_context().donated_input_ids = {0}
expected = graph_module(inp.clone())
clone_cleanup_pass(graph_module.graph)
graph_module.recompile()
# Clone should be removed since input is donated
assert count_clones(graph_module.graph) == 0
# Input can be mutated (donated)
inp_copy = inp.clone()
actual = graph_module(inp_copy)
torch.testing.assert_close(actual, expected)
def test_non_donated_input_clone_kept(self, clone_cleanup_pass):
"""Clone of non-donated input with mutation should be kept."""
def f(x: torch.Tensor, y: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
x_clone = x.clone()
x_clone.add_(1)
return x, x_clone
inp_x = torch.randn(2, 3)
inp_y = torch.randn(2, 3)
graph_module = make_fx(f)(inp_x, inp_y)
assert count_clones(graph_module.graph) == 1
# No donated inputs
get_pass_context().donated_input_ids = set()
expected = graph_module(inp_x.clone(), inp_y.clone())
clone_cleanup_pass(graph_module.graph)
graph_module.recompile()
# Clone should be kept since input is not donated and original is used
assert count_clones(graph_module.graph) == 1
# Verify inputs are not mutated
inp_x_before = inp_x.clone()
inp_y_before = inp_y.clone()
actual = graph_module(inp_x, inp_y)
torch.testing.assert_close(
inp_x, inp_x_before, msg="Input x should not be mutated"
)
torch.testing.assert_close(
inp_y, inp_y_before, msg="Input y should not be mutated"
)
torch.testing.assert_close(actual[0], expected[0])
torch.testing.assert_close(actual[1], expected[1])
def test_mixed_donated_inputs(self, clone_cleanup_pass):
"""Test with some inputs donated and some not."""
def f(x: torch.Tensor, y: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
x_clone = x.clone()
x_clone.add_(1)
y_clone = y.clone()
y_clone.add_(2)
return x_clone, y_clone
inp_x = torch.randn(2, 3)
inp_y = torch.randn(2, 3)
graph_module = make_fx(f)(inp_x, inp_y)
assert count_clones(graph_module.graph) == 2
# Only x is donated
get_pass_context().donated_input_ids = {0}
expected = graph_module(inp_x.clone(), inp_y.clone())
clone_cleanup_pass(graph_module.graph)
graph_module.recompile()
# x_clone removed (x is donated), y_clone kept (y is not donated)
assert count_clones(graph_module.graph) == 1
# Verify y is not mutated (x can be mutated since it's donated)
inp_y_before = inp_y.clone()
actual = graph_module(inp_x.clone(), inp_y)
torch.testing.assert_close(
inp_y, inp_y_before, msg="Input y should not be mutated"
)
torch.testing.assert_close(actual[0], expected[0])
torch.testing.assert_close(actual[1], expected[1])
@@ -0,0 +1,465 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Tests for IR inplace functionalization pass integration.
This test suite verifies that the inplace functionalization pass, lowering pass,
and clone cleanup pass work together correctly with donated buffer tracking.
"""
from collections.abc import Callable
import pytest
import torch
import torch._dynamo.exc
from torch import nn
import vllm.kernels # noqa: F401 to register kernels
from vllm.compilation.passes.inductor_pass import InductorPass, get_pass_context
from vllm.compilation.passes.ir.clone_elimination import (
UnsafeCloneEliminationPass,
)
from vllm.compilation.passes.ir.inplace_functionalization import (
VllmIRInplaceFunctionalizationPass,
)
from vllm.compilation.passes.ir.lowering_pass import VllmIRLoweringPass
from vllm.config import VllmConfig
from vllm.ir import ops
from vllm.platforms import current_platform
from vllm.triton_utils import HAS_TRITON, tl, triton
from ...backend import TestBackend
class StoreDonationInfoPass(InductorPass):
def __init__(self):
self.donated_input_ids_sets: list[set[int]] = []
def __call__(self, *args, **kwargs):
ctx = get_pass_context()
self.donated_input_ids_sets += [ctx.donated_input_ids]
class MaybeInplaceModel(nn.Module):
"""Model using only maybe_inplace variants."""
def __init__(self, hidden_size=16):
super().__init__()
self.weight1 = nn.Parameter(torch.ones(hidden_size, dtype=torch.bfloat16))
self.weight2 = nn.Parameter(torch.ones(hidden_size, dtype=torch.bfloat16))
def forward(
self, x: torch.Tensor, residual1: torch.Tensor, residual2: torch.Tensor
):
# First maybe_inplace - x & residual1 are donated
x_normed1, residual_out1 = ops.fused_add_rms_norm.maybe_inplace(
x, residual1, self.weight1, 1e-5
)
# Second maybe_inplace - residual2 is donated
x_normed2, residual_out2 = ops.fused_add_rms_norm.maybe_inplace(
x_normed1, residual2, self.weight2, 1e-5
)
return x_normed2, residual_out1, residual_out2
class FunctionalModel(nn.Module):
"""Model using only functional (default) variants."""
def __init__(self, hidden_size=16):
super().__init__()
self.weight1 = nn.Parameter(torch.ones(hidden_size, dtype=torch.bfloat16))
self.weight2 = nn.Parameter(torch.ones(hidden_size, dtype=torch.bfloat16))
def forward(
self, x: torch.Tensor, residual1: torch.Tensor, residual2: torch.Tensor
):
# First functional - no donation
x_normed1, residual_out1 = ops.fused_add_rms_norm(
x, residual1, self.weight1, 1e-5
)
# Second functional - no donation
x_normed2, residual_out2 = ops.fused_add_rms_norm(
x_normed1, residual2, self.weight2, 1e-5
)
return x_normed2, residual_out1, residual_out2
class MixedModel(nn.Module):
"""Model mixing maybe_inplace and functional variants."""
def __init__(self, hidden_size=16):
super().__init__()
self.weight1 = nn.Parameter(torch.ones(hidden_size, dtype=torch.bfloat16))
self.weight2 = nn.Parameter(torch.ones(hidden_size, dtype=torch.bfloat16))
def forward(
self, x: torch.Tensor, residual1: torch.Tensor, residual2: torch.Tensor
):
# First maybe_inplace - x & residual1 are donated
x_normed1, residual_out1 = ops.fused_add_rms_norm.maybe_inplace(
x, residual1, self.weight1, 1e-5
)
# Second functional - no donation, x_normed1 must be preserved as it's returned
x_normed2, residual_out2 = ops.fused_add_rms_norm(
x_normed1, residual2, self.weight2, 1e-5
)
# Return both to prevent x_normed1 from being optimized away
return x_normed1, x_normed2, residual_out1, residual_out2
class ModelWithTritonAfterMaybeInplace(nn.Module):
"""
Model using maybe_inplace followed by a Triton kernel.
Test clone elimination can handle Triton in the graph
"""
def __init__(self, hidden_size=16):
super().__init__()
self.weight = nn.Parameter(torch.ones(hidden_size, dtype=torch.bfloat16))
@triton.jit
def _triton_add_kernel(
x_ptr,
y_ptr,
n_elements,
BLOCK_SIZE: tl.constexpr,
):
pid = tl.program_id(axis=0)
block_start = pid * BLOCK_SIZE
offsets = block_start + tl.arange(0, BLOCK_SIZE)
mask = offsets < n_elements
x = tl.load(x_ptr + offsets, mask=mask)
y = x + 0.1
tl.store(y_ptr + offsets, y, mask=mask)
def triton_add(x: torch.Tensor) -> torch.Tensor:
"""Simple Triton add kernel."""
y = torch.empty_like(x)
n_elements = x.numel()
grid = (triton.cdiv(n_elements, 256),)
_triton_add_kernel[grid](x, y, n_elements, BLOCK_SIZE=256)
return y
self.triton_add = triton_add
def forward(self, x: torch.Tensor, residual: torch.Tensor, residual2: torch.Tensor):
x_normed, residual_out = ops.fused_add_rms_norm.maybe_inplace(
x, residual, self.weight, 1e-5
)
x_processed = self.triton_add(x_normed)
# x_processed does not need to be cloned, residual2 does
x_normed2, residual_out2 = ops.fused_add_rms_norm(
x_processed, residual2, self.weight, 1e-5
)
return x_normed2, residual_out2
skipif_no_triton = pytest.mark.skipif(not HAS_TRITON, reason="Requires Triton")
@pytest.mark.skipif(
not current_platform.is_cuda_alike(),
reason="Only test on cuda and rocm platform",
)
@pytest.mark.parametrize(
"model_class,expected_functionalized,expected_donated,expected_clones",
[
# 2 inplace calls, all activations donated, all clones eliminated
(MaybeInplaceModel, 2, 3, 0),
# No inplace calls, no donations, 3 clones (one eliminated)
(FunctionalModel, 0, 0, 3),
# One inplace call, two donated activations, 2 clones
(MixedModel, 1, 2, 2),
# One inplace call, two donated, 1 clone remaining
pytest.param(ModelWithTritonAfterMaybeInplace, 1, 2, 1, marks=skipif_no_triton),
],
)
def test_inplace_functionalization(
default_vllm_config: VllmConfig,
model_class,
expected_functionalized: int,
expected_clones: int,
expected_donated: int,
):
"""Test inplace functionalization, lowering, and clone cleanup."""
torch.set_default_device(current_platform.device_type)
# Use vllm_c so inplace path is triggered
default_vllm_config.kernel_config.ir_op_priority.fused_add_rms_norm = [
"vllm_c",
"native",
]
# Create passes in order they run during compilation
functionalization_pass = VllmIRInplaceFunctionalizationPass(default_vllm_config)
lowering_pass = VllmIRLoweringPass(default_vllm_config)
donated_info_pass = StoreDonationInfoPass()
cleanup_pass = UnsafeCloneEliminationPass(default_vllm_config)
# Set up backend with pre-grad pass
backend = TestBackend(lowering_pass, donated_info_pass, cleanup_pass)
backend.inductor_config["pre_grad_custom_pass"] = functionalization_pass
model = model_class()
x = torch.randn(8, 16, dtype=torch.bfloat16)
residual1 = torch.randn(8, 16, dtype=torch.bfloat16)
residual2 = torch.randn(8, 16, dtype=torch.bfloat16)
with default_vllm_config.kernel_config.ir_op_priority.set_priority():
# Reference output without optimization
ref_output = model(x.clone(), residual1.clone(), residual2.clone())
# Compile with inplace optimization
compiled_model = torch.compile(model, backend=backend, fullgraph=True)
output = compiled_model(x.clone(), residual1.clone(), residual2.clone())
# Verify correctness (relaxed tolerance for bfloat16)
for i in range(len(ref_output)):
torch.testing.assert_close(output[i], ref_output[i], rtol=1e-2, atol=1e-2)
# Verify expected number of ops were functionalized
func_ops = functionalization_pass.functionalized_ops
assert len(func_ops) == int(bool(expected_functionalized))
if expected_functionalized > 0:
assert "fused_add_rms_norm" in func_ops
assert func_ops["fused_add_rms_norm"] == expected_functionalized
# Verify lowering happened (2 ops in all cases)
assert "fused_add_rms_norm" in lowering_pass.selected_impls
assert len(lowering_pass.selected_impls["fused_add_rms_norm"]) == 2
assert all(
provider == "vllm_c"
for node, provider in lowering_pass.selected_impls["fused_add_rms_norm"].items()
), lowering_pass.selected_impls
# Verify correct number of donated IDs
assert len(donated_info_pass.donated_input_ids_sets) == 1
assert len(donated_info_pass.donated_input_ids_sets[0]) == expected_donated
# Verify expected number of clones after cleanup
actual_clones = backend.op_count(torch.ops.aten.clone.default, before=False)
assert actual_clones == expected_clones, (
f"Expected {expected_clones} clones, got {actual_clones}:"
f"{backend.print_graphs()}"
)
@pytest.mark.skipif(
not current_platform.is_cuda_alike(),
reason="Only test on cuda and rocm platform",
)
def test_donated_buffer_context_propagation(default_vllm_config):
"""Test that donated_input_ids propagates correctly through pass_context."""
torch.set_default_device(current_platform.device_type)
# Create a custom backend that inspects pass_context in cleanup pass
functionalization_pass = VllmIRInplaceFunctionalizationPass(default_vllm_config)
lowering_pass = VllmIRLoweringPass(default_vllm_config)
donation_info_pass = StoreDonationInfoPass()
cleanup_pass = UnsafeCloneEliminationPass(default_vllm_config)
backend = TestBackend(lowering_pass, donation_info_pass, cleanup_pass)
backend.inductor_config["pre_grad_custom_pass"] = functionalization_pass
model = MaybeInplaceModel()
x = torch.randn(8, 16, dtype=torch.bfloat16)
residual1 = torch.randn(8, 16, dtype=torch.bfloat16)
residual2 = torch.randn(8, 16, dtype=torch.bfloat16)
compiled_model = torch.compile(model, backend=backend, fullgraph=True)
compiled_model(x.clone(), residual1.clone(), residual2.clone())
donated_ids_seen = donation_info_pass.donated_input_ids_sets
# Verify donated_input_ids was set and propagated
assert len(donated_ids_seen) == 1
# Should have donated inputs (exact indices depend on AOTAutograd)
assert len(donated_ids_seen[0]) == 3
# All donated ids should be valid non-negative integers
for idx in donated_ids_seen[0]:
assert isinstance(idx, int) and idx >= 0, f"Invalid donated index: {idx}"
@pytest.mark.skipif(
not current_platform.is_cuda_alike(),
reason="Only test on cuda and rocm platform",
)
def test_maybe_inplace_reuse_error(default_vllm_config):
"""Test that reusing a donated activation input raises ValueError."""
torch.set_default_device(current_platform.device_type)
class ReuseModel(nn.Module):
"""Model that incorrectly reuses a donated activation input."""
def __init__(self, hidden_size=16):
super().__init__()
self.weight = nn.Parameter(torch.ones(hidden_size, dtype=torch.bfloat16))
def forward(self, x: torch.Tensor, residual: torch.Tensor):
# x is donated to maybe_inplace
x_normed, residual_out = ops.fused_add_rms_norm.maybe_inplace(
x, residual, self.weight, 1e-5
)
# ERROR: x is used again after being donated
return x_normed + x # This should raise ValueError
functionalization_pass = VllmIRInplaceFunctionalizationPass(default_vllm_config)
lowering_pass = VllmIRLoweringPass(default_vllm_config)
cleanup_pass = UnsafeCloneEliminationPass(default_vllm_config)
backend = TestBackend(lowering_pass, cleanup_pass)
backend.inductor_config["pre_grad_custom_pass"] = functionalization_pass
model = ReuseModel()
x = torch.randn(8, 16, dtype=torch.bfloat16)
residual = torch.randn(8, 16, dtype=torch.bfloat16)
# Compilation should raise BackendCompilerFailed wrapping ValueError
with pytest.raises(
torch._dynamo.exc.BackendCompilerFailed,
match="is used again after the node",
):
compiled_model = torch.compile(model, backend=backend, fullgraph=True)
compiled_model(x.clone(), residual.clone())
# Piecewise compilation tests with graph splitting
@torch.library.custom_op("vllm::test_split_marker", mutates_args=())
def test_split_marker(x: torch.Tensor) -> torch.Tensor:
"""Identity op that marks a split point for piecewise compilation."""
return x.clone()
@test_split_marker.register_fake
def _fake_split_marker(x: torch.Tensor) -> torch.Tensor:
return torch.empty_like(x)
class TransformerBlockWithSplits(nn.Module):
"""Transformer block with explicit split points for piecewise compilation."""
def __init__(self, hidden_size=32, intermediate_size=128):
super().__init__()
self.hidden_size = hidden_size
self.intermediate_size = intermediate_size
# Attention-like projection
self.attn_proj = nn.Linear(
hidden_size, hidden_size, bias=False, dtype=torch.bfloat16
)
# Post-attention norm
self.post_attn_norm = nn.Parameter(
torch.ones(hidden_size, dtype=torch.bfloat16)
)
# MLP
self.gate_proj = nn.Linear(
hidden_size, intermediate_size, bias=False, dtype=torch.bfloat16
)
self.up_proj = nn.Linear(
hidden_size, intermediate_size, bias=False, dtype=torch.bfloat16
)
self.down_proj = nn.Linear(
intermediate_size, hidden_size, bias=False, dtype=torch.bfloat16
)
# Post-MLP norm
self.post_mlp_norm = nn.Parameter(torch.ones(hidden_size, dtype=torch.bfloat16))
def forward(self, x: torch.Tensor):
# Attention block with residual
residual1 = x
attn_out = self.attn_proj(x)
# Fused add + norm (maybe_inplace: residual1 is donated)
normed1, residual1 = ops.fused_add_rms_norm.maybe_inplace(
attn_out, residual1, self.post_attn_norm, 1e-5
)
# Force a graph split here
normed1 = torch.ops.vllm.test_split_marker(normed1)
# MLP block
gate = self.gate_proj(normed1)
up = self.up_proj(normed1)
mlp_out = self.down_proj(gate * torch.nn.functional.silu(up))
# Fused add + norm (maybe_inplace: residual1 is donated)
normed2, residual2 = ops.fused_add_rms_norm.maybe_inplace(
mlp_out, residual1, self.post_mlp_norm, 1e-5
)
return normed2, residual2
def with_dyn_arg(fn: Callable, arg_index: int, dim_index: int):
def inner(*args):
torch._dynamo.mark_dynamic(args[arg_index], dim_index)
return fn(*args)
return inner
@pytest.mark.skipif(
not current_platform.is_cuda_alike(),
reason="Only test on cuda and rocm platform",
)
def test_piecewise_compilation_with_donated_buffers(monkeypatch, fresh_vllm_cache):
"""
Test piecewise compilation with donated buffers across graph splits.
Utilizes a custom splitting op. Uses fresh cache to avoid compilation caching.
"""
torch.set_default_device(current_platform.device_type)
# Disable compilation cache to avoid serialization issues
monkeypatch.setenv("VLLM_DISABLE_COMPILE_CACHE", "1")
from vllm.compilation.backends import VllmBackend
from vllm.config import CompilationConfig, VllmConfig
# Create config with custom splitting op
store_donation_info = StoreDonationInfoPass()
vllm_config = VllmConfig(
compilation_config=CompilationConfig(
custom_ops=["all"],
splitting_ops=["vllm::test_split_marker"],
inductor_compile_config={"post_grad_custom_post_pass": store_donation_info},
)
)
backend = VllmBackend(vllm_config)
model = TransformerBlockWithSplits()
x = torch.randn(8, 32, dtype=torch.bfloat16)
# Reference output
ref_output = with_dyn_arg(model, 0, 0)(x.clone())
# Compile with piecewise compilation (graph will split at split_marker)
compiled_model = torch.compile(model, backend=backend, fullgraph=False)
output = with_dyn_arg(compiled_model, 0, 0)(x.clone())
# Verify correctness (relaxed tolerance for bfloat16)
torch.testing.assert_close(output[0], ref_output[0], rtol=1e-2, atol=1e-2)
torch.testing.assert_close(output[1], ref_output[1], rtol=1e-2, atol=1e-2)
# Verify the model was split into multiple submodules
assert hasattr(backend, "split_gm"), "Backend should have split graph module"
# Should have at least 2 submodules (split by test_split_marker op)
submodules = list(backend.split_gm.named_children())
num_submodules = len(submodules)
assert num_submodules >= 2, (
f"Expected at least 2 submodules (split), got {num_submodules}"
)
# Check that donation info was propagated correctly
donated_inputs_sets = store_donation_info.donated_input_ids_sets
assert len(donated_inputs_sets) == 2
assert len(donated_inputs_sets[0]) == 1
assert len(donated_inputs_sets[1]) == 1
@@ -126,7 +126,7 @@ class TestFusedAddRMSNorm(torch.nn.Module):
if TEST_FP8 and do_fusion:
return [torch.ops._C.fused_add_rms_norm_static_fp8_quant.default]
else:
return [torch.ops._C.fused_add_rms_norm.default]
return []
def ops_not_in_model(self):
return []
@@ -59,7 +59,7 @@ class TestModel(torch.nn.Module):
def ops_in_model_before(self):
return [
rocm_aiter_ops.get_rmsnorm_fused_add_op(),
torch.ops.vllm_ir.fused_add_rms_norm,
torch.ops.aten.constant_pad_nd,
]
+4 -15
View File
@@ -17,7 +17,6 @@ from vllm.compilation.passes.fusion.rms_quant_fusion import (
FusedRMSQuantKey,
RMSNormQuantFusionPass,
)
from vllm.compilation.passes.fx_utils import find_op_nodes
from vllm.compilation.passes.utility.noop_elimination import NoOpEliminationPass
from vllm.compilation.passes.utility.post_cleanup import PostCleanupPass
from vllm.config import (
@@ -243,9 +242,10 @@ class TestModel(torch.nn.Module):
]
def ops_in_model_before_partial(self):
return [torch.ops.vllm_ir.rms_norm] + (
[RMS_ADD_OP] if self.enable_rms_norm_custom_op else [torch.ops.aten.rsqrt]
)
return [
torch.ops.vllm_ir.rms_norm,
torch.ops.vllm_ir.fused_add_rms_norm.default,
]
def _run_fusion_test(
@@ -383,17 +383,6 @@ def test_fusion_rmsnorm_quant(
model.ops_in_model_before_partial(), fully_replaced=False
)
# If RMSNorm custom op is disabled (native/torch impl used),
# there's a risk that the fused add doesn't get included in the
# replacement and only the rms part gets fused with quant.
# Hence, we check only 2 add nodes are left (final fused rmsnorm add).
if not enable_rms_norm_custom_op:
n_add_nodes = lambda g: sum(1 for _ in find_op_nodes(torch.ops.aten.add, g))
# rms_norm is IR, not included
# 6 = 3x2 (3xRMS_ADD, 2 each)
assert n_add_nodes(backend.graph_pre_pass) == 6
assert n_add_nodes(backend.graph_post_pass) == 2
@pytest.mark.parametrize("dtype", [torch.bfloat16])
@pytest.mark.parametrize("hidden_size", [256])