[vLLM IR] 2/N fused_add_rms_norm and maybe_inplace overload (#36823)
Signed-off-by: Luka Govedič <lgovedic@redhat.com> Signed-off-by: Luka Govedič <ProExpertProg@users.noreply.github.com>
This commit is contained in:
@@ -12,10 +12,17 @@ from torch._ops import OpOverload, OpOverloadPacket
|
||||
from torch.fx._utils import lazy_format_graph_code
|
||||
|
||||
from vllm.compilation.passes.fx_utils import find_op_nodes
|
||||
from vllm.compilation.passes.inductor_pass import InductorPass
|
||||
from vllm.compilation.passes.inductor_pass import (
|
||||
InductorPass,
|
||||
pass_context,
|
||||
)
|
||||
from vllm.compilation.passes.ir.inplace_functionalization import (
|
||||
VllmIRInplaceFunctionalizationPass,
|
||||
)
|
||||
from vllm.compilation.passes.pass_manager import with_pattern_match_debug
|
||||
from vllm.compilation.passes.vllm_inductor_pass import VllmInductorPass
|
||||
from vllm.config import VllmConfig, get_current_vllm_config
|
||||
from vllm.config.utils import Range
|
||||
from vllm.logger import init_logger
|
||||
|
||||
logger = init_logger("vllm.tests.compile.backend")
|
||||
@@ -53,11 +60,17 @@ class TestBackend:
|
||||
self.custom_passes = list(passes)
|
||||
vllm_config = get_current_vllm_config()
|
||||
compile_config = vllm_config.compilation_config
|
||||
self.range = Range(1, vllm_config.scheduler_config.max_num_batched_tokens)
|
||||
# Deepcopy to allow multiple TestBackend instances to use the same VllmConfig
|
||||
self.inductor_config = deepcopy(compile_config.inductor_compile_config)
|
||||
self.inductor_config["force_disable_caches"] = True
|
||||
self.inductor_config["post_grad_custom_post_pass"] = self.post_pass
|
||||
|
||||
# Add VllmIRInplaceFunctionalizationPass as pre-grad pass by default
|
||||
self.inductor_config["pre_grad_custom_pass"] = (
|
||||
VllmIRInplaceFunctionalizationPass(vllm_config)
|
||||
)
|
||||
|
||||
if debug_dump_path := vllm_config.compile_debug_dump_path():
|
||||
logger.debug("Dumping depyf output to %s", debug_dump_path)
|
||||
self.debug_ctx = depyf.prepare_debug(debug_dump_path.as_posix())
|
||||
@@ -68,7 +81,7 @@ class TestBackend:
|
||||
self.graph_pre_compile = deepcopy(graph)
|
||||
from torch._inductor.compile_fx import compile_fx
|
||||
|
||||
with self.debug_ctx:
|
||||
with self.debug_ctx, pass_context(self.range):
|
||||
return compile_fx(
|
||||
graph, example_inputs, config_patches=self.inductor_config
|
||||
)
|
||||
|
||||
@@ -88,14 +88,10 @@ class TestAllReduceRMSNormModel(torch.nn.Module):
|
||||
]
|
||||
|
||||
def ops_in_model(self):
|
||||
return (
|
||||
[torch.ops.vllm_ir.rms_norm]
|
||||
+ [
|
||||
torch.ops._C.fused_add_rms_norm.default,
|
||||
]
|
||||
if RMSNorm.enabled()
|
||||
else []
|
||||
)
|
||||
return [
|
||||
torch.ops.vllm_ir.rms_norm,
|
||||
torch.ops.vllm_ir.fused_add_rms_norm,
|
||||
]
|
||||
|
||||
|
||||
class TestAllReduceRMSNormStaticQuantFP8Model(torch.nn.Module):
|
||||
@@ -152,16 +148,17 @@ class TestAllReduceRMSNormStaticQuantFP8Model(torch.nn.Module):
|
||||
def ops_in_model(self):
|
||||
if self.vllm_config.compilation_config.pass_config.fuse_norm_quant:
|
||||
return [torch.ops._C.fused_add_rms_norm_static_fp8_quant.default]
|
||||
elif RMSNorm.enabled():
|
||||
return [
|
||||
torch.ops._C.fused_add_rms_norm.default,
|
||||
]
|
||||
elif any(layer.is_quant_fp8_enabled() for layer in self.fp8_linear_layers):
|
||||
return [
|
||||
torch.ops._C.static_scaled_fp8_quant.default,
|
||||
]
|
||||
else:
|
||||
return []
|
||||
quant_ops = (
|
||||
[torch.ops._C.static_scaled_fp8_quant.default]
|
||||
if any(layer.is_quant_fp8_enabled() for layer in self.fp8_linear_layers)
|
||||
else [torch.ops.aten.reciprocal]
|
||||
)
|
||||
return [
|
||||
torch.ops.vllm_ir.rms_norm,
|
||||
torch.ops.vllm_ir.fused_add_rms_norm,
|
||||
*quant_ops,
|
||||
]
|
||||
|
||||
|
||||
@multi_gpu_test(num_gpus=2)
|
||||
|
||||
@@ -0,0 +1,412 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Comprehensive tests for UnsafeCloneEliminationPass.
|
||||
|
||||
This test suite exercises all possible valid FX graph patterns involving clones:
|
||||
1. Clone with no users (dead code)
|
||||
2. Clone with read-only users
|
||||
3. Clone with mutation users
|
||||
4. Clone of graph input
|
||||
5. Clone with original used after mutation
|
||||
6. Clone chains
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from torch import fx
|
||||
from torch.fx.experimental.proxy_tensor import make_fx
|
||||
|
||||
from vllm.compilation.passes.fx_utils import find_op_nodes
|
||||
from vllm.compilation.passes.inductor_pass import get_pass_context, pass_context
|
||||
from vllm.compilation.passes.ir.clone_elimination import (
|
||||
UnsafeCloneEliminationPass,
|
||||
user_writes_to_node,
|
||||
)
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.config.utils import Range
|
||||
|
||||
|
||||
def count_clones(graph: fx.Graph) -> int:
|
||||
"""Count clone nodes in a graph."""
|
||||
return len(list(find_op_nodes(torch.ops.aten.clone.default, graph)))
|
||||
|
||||
|
||||
@pytest.fixture(scope="function")
|
||||
def clone_cleanup_pass():
|
||||
return UnsafeCloneEliminationPass(VllmConfig())
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def setup_pass_context():
|
||||
"""Set up pass context for each test."""
|
||||
with pass_context(compile_range=Range(1, 8192)):
|
||||
yield
|
||||
|
||||
|
||||
class TestCloneCleanup:
|
||||
"""Test UnsafeCloneEliminationPass behavior on various graph patterns."""
|
||||
|
||||
def test_remove_clone_readonly_users(self, clone_cleanup_pass):
|
||||
"""Clone with only read-only users should be removed."""
|
||||
|
||||
def f(x: torch.Tensor) -> torch.Tensor:
|
||||
x_clone = x.clone()
|
||||
return x_clone + 1
|
||||
|
||||
inp = torch.randn(2, 3)
|
||||
graph_module = make_fx(f)(inp)
|
||||
assert count_clones(graph_module.graph) == 1
|
||||
|
||||
expected = graph_module(inp)
|
||||
clone_cleanup_pass(graph_module.graph)
|
||||
graph_module.recompile()
|
||||
actual = graph_module(inp)
|
||||
|
||||
assert count_clones(graph_module.graph) == 0
|
||||
torch.testing.assert_close(actual, expected)
|
||||
|
||||
def test_keep_clone_with_mutation_and_original_used_after(self, clone_cleanup_pass):
|
||||
"""Clone must be kept if it's mutated AND original is used after mutation."""
|
||||
|
||||
def f(x: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
x = x.relu() # not a graph param
|
||||
x_clone = x.clone()
|
||||
x_clone.add_(1)
|
||||
return x, x_clone
|
||||
|
||||
inp = torch.randn(2, 3)
|
||||
graph_module = make_fx(f)(inp)
|
||||
assert count_clones(graph_module.graph) == 1
|
||||
|
||||
expected = graph_module(inp)
|
||||
clone_cleanup_pass(graph_module.graph)
|
||||
graph_module.recompile()
|
||||
actual = graph_module(inp)
|
||||
|
||||
# Clone should be KEPT because original is used after mutation
|
||||
assert count_clones(graph_module.graph) == 1
|
||||
torch.testing.assert_close(actual[0], expected[0])
|
||||
torch.testing.assert_close(actual[1], expected[1])
|
||||
|
||||
def test_remove_clone_with_mutation_no_original_use(self, clone_cleanup_pass):
|
||||
"""Clone can be removed if it's mutated but original is not used after."""
|
||||
|
||||
def f(x: torch.Tensor) -> torch.Tensor:
|
||||
x = x.relu() # not a graph param
|
||||
x_clone = x.clone()
|
||||
x_clone.add_(1)
|
||||
return x_clone
|
||||
|
||||
inp = torch.randn(2, 3)
|
||||
graph_module = make_fx(f)(inp)
|
||||
assert count_clones(graph_module.graph) == 1
|
||||
|
||||
expected = graph_module(inp)
|
||||
clone_cleanup_pass(graph_module.graph)
|
||||
graph_module.recompile()
|
||||
actual = graph_module(inp)
|
||||
|
||||
assert count_clones(graph_module.graph) == 0
|
||||
torch.testing.assert_close(actual, expected)
|
||||
|
||||
def test_clone_chain(self, clone_cleanup_pass):
|
||||
"""Test handling of clone chains: x -> clone1 -> clone2."""
|
||||
|
||||
def f(x: torch.Tensor) -> torch.Tensor:
|
||||
x = x.relu() # not a graph param
|
||||
x1 = x.clone()
|
||||
x2 = x1.clone()
|
||||
return x2 + 1
|
||||
|
||||
inp = torch.randn(2, 3)
|
||||
graph_module = make_fx(f)(inp)
|
||||
assert count_clones(graph_module.graph) == 2
|
||||
|
||||
expected = graph_module(inp)
|
||||
clone_cleanup_pass(graph_module.graph)
|
||||
graph_module.recompile()
|
||||
actual = graph_module(inp)
|
||||
|
||||
# Both clones should be removed
|
||||
assert count_clones(graph_module.graph) == 0
|
||||
torch.testing.assert_close(actual, expected)
|
||||
|
||||
def test_multiple_clones_of_same_input(self, clone_cleanup_pass):
|
||||
"""Test multiple independent clones of the same input."""
|
||||
|
||||
def f(x: torch.Tensor) -> torch.Tensor:
|
||||
x1 = x.clone()
|
||||
x2 = x.clone()
|
||||
return x1 + x2
|
||||
|
||||
inp = torch.randn(2, 3)
|
||||
graph_module = make_fx(f)(inp)
|
||||
assert count_clones(graph_module.graph) == 2
|
||||
|
||||
expected = graph_module(inp)
|
||||
clone_cleanup_pass(graph_module.graph)
|
||||
graph_module.recompile()
|
||||
actual = graph_module(inp)
|
||||
|
||||
# Both clones should be removed (only readonly uses)
|
||||
assert count_clones(graph_module.graph) == 0
|
||||
torch.testing.assert_close(actual, expected)
|
||||
|
||||
def test_no_clones_in_graph(self, clone_cleanup_pass):
|
||||
"""Test pass behavior when graph has no clones."""
|
||||
|
||||
def f(x: torch.Tensor) -> torch.Tensor:
|
||||
return x + 1
|
||||
|
||||
inp = torch.randn(2, 3)
|
||||
graph_module = make_fx(f)(inp)
|
||||
assert count_clones(graph_module.graph) == 0
|
||||
|
||||
expected = graph_module(inp)
|
||||
clone_cleanup_pass(graph_module.graph)
|
||||
graph_module.recompile()
|
||||
actual = graph_module(inp)
|
||||
|
||||
assert count_clones(graph_module.graph) == 0
|
||||
torch.testing.assert_close(actual, expected)
|
||||
|
||||
def test_multiple_passes(self, clone_cleanup_pass):
|
||||
"""Test running the pass multiple times (should be idempotent)."""
|
||||
|
||||
def f(x: torch.Tensor) -> torch.Tensor:
|
||||
x1 = x.clone()
|
||||
return x1 + 1
|
||||
|
||||
inp = torch.randn(2, 3)
|
||||
graph_module = make_fx(f)(inp)
|
||||
assert count_clones(graph_module.graph) == 1
|
||||
|
||||
expected = graph_module(inp)
|
||||
|
||||
clone_cleanup_pass(graph_module.graph)
|
||||
assert count_clones(graph_module.graph) == 0
|
||||
graph_module.recompile()
|
||||
actual = graph_module(inp)
|
||||
torch.testing.assert_close(actual, expected)
|
||||
|
||||
clone_cleanup_pass(graph_module.graph)
|
||||
assert count_clones(graph_module.graph) == 0
|
||||
graph_module.recompile()
|
||||
actual = graph_module(inp)
|
||||
torch.testing.assert_close(actual, expected)
|
||||
|
||||
def test_output_node_no_write(self):
|
||||
"""Output nodes never write to their inputs."""
|
||||
|
||||
def f(x: torch.Tensor) -> torch.Tensor:
|
||||
return x
|
||||
|
||||
graph_module = make_fx(f)(torch.randn(2, 3))
|
||||
x_node = [n for n in graph_module.graph.nodes if n.op == "placeholder"][0]
|
||||
output_node = [n for n in graph_module.graph.nodes if n.op == "output"][0]
|
||||
|
||||
assert not user_writes_to_node(output_node, x_node)
|
||||
|
||||
def test_readonly_op_no_write(self):
|
||||
"""Readonly operations don't write to inputs."""
|
||||
|
||||
def f(x: torch.Tensor, y: torch.Tensor) -> torch.Tensor:
|
||||
return x + y
|
||||
|
||||
graph_module = make_fx(f)(torch.randn(2, 3), torch.randn(2, 3))
|
||||
placeholders = [n for n in graph_module.graph.nodes if n.op == "placeholder"]
|
||||
add_node = [
|
||||
n
|
||||
for n in graph_module.graph.nodes
|
||||
if n.op == "call_function" and n.target == torch.ops.aten.add.Tensor
|
||||
][0]
|
||||
|
||||
assert not user_writes_to_node(add_node, placeholders[0])
|
||||
assert not user_writes_to_node(add_node, placeholders[1])
|
||||
|
||||
def test_inplace_op_writes(self):
|
||||
"""Inplace operations write to first argument."""
|
||||
|
||||
def f(x: torch.Tensor, y: torch.Tensor) -> torch.Tensor:
|
||||
x.add_(y)
|
||||
return x
|
||||
|
||||
graph_module = make_fx(f)(torch.randn(2, 3), torch.randn(2, 3))
|
||||
placeholders = [n for n in graph_module.graph.nodes if n.op == "placeholder"]
|
||||
add_node = [
|
||||
n
|
||||
for n in graph_module.graph.nodes
|
||||
if n.op == "call_function" and "add_" in str(n.target)
|
||||
][0]
|
||||
|
||||
# add_ writes to first arg but not second
|
||||
assert user_writes_to_node(add_node, placeholders[0])
|
||||
assert not user_writes_to_node(add_node, placeholders[1])
|
||||
|
||||
def test_copy_writes(self):
|
||||
"""copy_ operation writes to first argument."""
|
||||
|
||||
def f(x: torch.Tensor, y: torch.Tensor) -> torch.Tensor:
|
||||
x.copy_(y)
|
||||
return x
|
||||
|
||||
graph_module = make_fx(f)(torch.randn(2, 3), torch.randn(2, 3))
|
||||
placeholders = [n for n in graph_module.graph.nodes if n.op == "placeholder"]
|
||||
copy_node = [
|
||||
n
|
||||
for n in graph_module.graph.nodes
|
||||
if n.op == "call_function" and "copy_" in str(n.target)
|
||||
][0]
|
||||
|
||||
assert user_writes_to_node(copy_node, placeholders[0])
|
||||
assert not user_writes_to_node(copy_node, placeholders[1])
|
||||
|
||||
def test_auto_functionalized_not_a_write(self):
|
||||
"""auto_functionalized ops are follow-up uses, not writes."""
|
||||
from torch._higher_order_ops.auto_functionalize import auto_functionalized
|
||||
|
||||
def f(x: torch.Tensor) -> torch.Tensor:
|
||||
return x
|
||||
|
||||
graph_module = make_fx(f)(torch.randn(2, 3))
|
||||
x_node = [n for n in graph_module.graph.nodes if n.op == "placeholder"][0]
|
||||
|
||||
# Create an auto_functionalized node in the graph
|
||||
with graph_module.graph.inserting_before(None):
|
||||
af_node = graph_module.graph.call_function(
|
||||
auto_functionalized, kwargs={"input": x_node}
|
||||
)
|
||||
|
||||
# auto_functionalized should not be treated as a write
|
||||
assert not user_writes_to_node(af_node, x_node)
|
||||
|
||||
def test_higher_order_op_conservatively_writes(self):
|
||||
"""Other higher-order operators are conservatively treated as writes."""
|
||||
from torch._ops import HigherOrderOperator
|
||||
|
||||
def f(x: torch.Tensor) -> torch.Tensor:
|
||||
return x
|
||||
|
||||
graph_module = make_fx(f)(torch.randn(2, 3))
|
||||
x_node = [n for n in graph_module.graph.nodes if n.op == "placeholder"][0]
|
||||
|
||||
# Create a concrete higher-order operator subclass
|
||||
class MockHigherOrderOp(HigherOrderOperator):
|
||||
def __call__(self, *args, **kwargs):
|
||||
return args[0] if args else None
|
||||
|
||||
mock_hoo = MockHigherOrderOp("mock_higher_order_op")
|
||||
|
||||
with graph_module.graph.inserting_before(None):
|
||||
hoo_node = graph_module.graph.call_function(mock_hoo, args=(x_node,))
|
||||
|
||||
# Should be conservative and assume it could write
|
||||
assert user_writes_to_node(hoo_node, x_node)
|
||||
|
||||
|
||||
class TestCloneCleanupWithDonatedInputs:
|
||||
"""Test UnsafeCloneEliminationPass with donated input tracking via PassContext."""
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def setup_pass_context(self):
|
||||
"""Set up pass context for each test."""
|
||||
with pass_context(compile_range=Range(1, 8192)):
|
||||
yield
|
||||
|
||||
def test_donated_input_clone_removed(self, clone_cleanup_pass):
|
||||
"""Clone of donated input should be removed."""
|
||||
|
||||
def f(x: torch.Tensor) -> torch.Tensor:
|
||||
x_clone = x.clone()
|
||||
x_clone.add_(1)
|
||||
return x_clone
|
||||
|
||||
inp = torch.randn(2, 3)
|
||||
graph_module = make_fx(f)(inp)
|
||||
assert count_clones(graph_module.graph) == 1
|
||||
|
||||
# Mark first parameter as donated
|
||||
get_pass_context().donated_input_ids = {0}
|
||||
|
||||
expected = graph_module(inp.clone())
|
||||
clone_cleanup_pass(graph_module.graph)
|
||||
graph_module.recompile()
|
||||
|
||||
# Clone should be removed since input is donated
|
||||
assert count_clones(graph_module.graph) == 0
|
||||
|
||||
# Input can be mutated (donated)
|
||||
inp_copy = inp.clone()
|
||||
actual = graph_module(inp_copy)
|
||||
torch.testing.assert_close(actual, expected)
|
||||
|
||||
def test_non_donated_input_clone_kept(self, clone_cleanup_pass):
|
||||
"""Clone of non-donated input with mutation should be kept."""
|
||||
|
||||
def f(x: torch.Tensor, y: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
x_clone = x.clone()
|
||||
x_clone.add_(1)
|
||||
return x, x_clone
|
||||
|
||||
inp_x = torch.randn(2, 3)
|
||||
inp_y = torch.randn(2, 3)
|
||||
graph_module = make_fx(f)(inp_x, inp_y)
|
||||
assert count_clones(graph_module.graph) == 1
|
||||
|
||||
# No donated inputs
|
||||
get_pass_context().donated_input_ids = set()
|
||||
|
||||
expected = graph_module(inp_x.clone(), inp_y.clone())
|
||||
clone_cleanup_pass(graph_module.graph)
|
||||
graph_module.recompile()
|
||||
|
||||
# Clone should be kept since input is not donated and original is used
|
||||
assert count_clones(graph_module.graph) == 1
|
||||
|
||||
# Verify inputs are not mutated
|
||||
inp_x_before = inp_x.clone()
|
||||
inp_y_before = inp_y.clone()
|
||||
actual = graph_module(inp_x, inp_y)
|
||||
torch.testing.assert_close(
|
||||
inp_x, inp_x_before, msg="Input x should not be mutated"
|
||||
)
|
||||
torch.testing.assert_close(
|
||||
inp_y, inp_y_before, msg="Input y should not be mutated"
|
||||
)
|
||||
torch.testing.assert_close(actual[0], expected[0])
|
||||
torch.testing.assert_close(actual[1], expected[1])
|
||||
|
||||
def test_mixed_donated_inputs(self, clone_cleanup_pass):
|
||||
"""Test with some inputs donated and some not."""
|
||||
|
||||
def f(x: torch.Tensor, y: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
x_clone = x.clone()
|
||||
x_clone.add_(1)
|
||||
y_clone = y.clone()
|
||||
y_clone.add_(2)
|
||||
return x_clone, y_clone
|
||||
|
||||
inp_x = torch.randn(2, 3)
|
||||
inp_y = torch.randn(2, 3)
|
||||
graph_module = make_fx(f)(inp_x, inp_y)
|
||||
assert count_clones(graph_module.graph) == 2
|
||||
|
||||
# Only x is donated
|
||||
get_pass_context().donated_input_ids = {0}
|
||||
|
||||
expected = graph_module(inp_x.clone(), inp_y.clone())
|
||||
clone_cleanup_pass(graph_module.graph)
|
||||
graph_module.recompile()
|
||||
|
||||
# x_clone removed (x is donated), y_clone kept (y is not donated)
|
||||
assert count_clones(graph_module.graph) == 1
|
||||
|
||||
# Verify y is not mutated (x can be mutated since it's donated)
|
||||
inp_y_before = inp_y.clone()
|
||||
actual = graph_module(inp_x.clone(), inp_y)
|
||||
torch.testing.assert_close(
|
||||
inp_y, inp_y_before, msg="Input y should not be mutated"
|
||||
)
|
||||
torch.testing.assert_close(actual[0], expected[0])
|
||||
torch.testing.assert_close(actual[1], expected[1])
|
||||
@@ -0,0 +1,465 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Tests for IR inplace functionalization pass integration.
|
||||
|
||||
This test suite verifies that the inplace functionalization pass, lowering pass,
|
||||
and clone cleanup pass work together correctly with donated buffer tracking.
|
||||
"""
|
||||
|
||||
from collections.abc import Callable
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
import torch._dynamo.exc
|
||||
from torch import nn
|
||||
|
||||
import vllm.kernels # noqa: F401 to register kernels
|
||||
from vllm.compilation.passes.inductor_pass import InductorPass, get_pass_context
|
||||
from vllm.compilation.passes.ir.clone_elimination import (
|
||||
UnsafeCloneEliminationPass,
|
||||
)
|
||||
from vllm.compilation.passes.ir.inplace_functionalization import (
|
||||
VllmIRInplaceFunctionalizationPass,
|
||||
)
|
||||
from vllm.compilation.passes.ir.lowering_pass import VllmIRLoweringPass
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.ir import ops
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import HAS_TRITON, tl, triton
|
||||
|
||||
from ...backend import TestBackend
|
||||
|
||||
|
||||
class StoreDonationInfoPass(InductorPass):
|
||||
def __init__(self):
|
||||
self.donated_input_ids_sets: list[set[int]] = []
|
||||
|
||||
def __call__(self, *args, **kwargs):
|
||||
ctx = get_pass_context()
|
||||
self.donated_input_ids_sets += [ctx.donated_input_ids]
|
||||
|
||||
|
||||
class MaybeInplaceModel(nn.Module):
|
||||
"""Model using only maybe_inplace variants."""
|
||||
|
||||
def __init__(self, hidden_size=16):
|
||||
super().__init__()
|
||||
self.weight1 = nn.Parameter(torch.ones(hidden_size, dtype=torch.bfloat16))
|
||||
self.weight2 = nn.Parameter(torch.ones(hidden_size, dtype=torch.bfloat16))
|
||||
|
||||
def forward(
|
||||
self, x: torch.Tensor, residual1: torch.Tensor, residual2: torch.Tensor
|
||||
):
|
||||
# First maybe_inplace - x & residual1 are donated
|
||||
x_normed1, residual_out1 = ops.fused_add_rms_norm.maybe_inplace(
|
||||
x, residual1, self.weight1, 1e-5
|
||||
)
|
||||
# Second maybe_inplace - residual2 is donated
|
||||
x_normed2, residual_out2 = ops.fused_add_rms_norm.maybe_inplace(
|
||||
x_normed1, residual2, self.weight2, 1e-5
|
||||
)
|
||||
return x_normed2, residual_out1, residual_out2
|
||||
|
||||
|
||||
class FunctionalModel(nn.Module):
|
||||
"""Model using only functional (default) variants."""
|
||||
|
||||
def __init__(self, hidden_size=16):
|
||||
super().__init__()
|
||||
self.weight1 = nn.Parameter(torch.ones(hidden_size, dtype=torch.bfloat16))
|
||||
self.weight2 = nn.Parameter(torch.ones(hidden_size, dtype=torch.bfloat16))
|
||||
|
||||
def forward(
|
||||
self, x: torch.Tensor, residual1: torch.Tensor, residual2: torch.Tensor
|
||||
):
|
||||
# First functional - no donation
|
||||
x_normed1, residual_out1 = ops.fused_add_rms_norm(
|
||||
x, residual1, self.weight1, 1e-5
|
||||
)
|
||||
# Second functional - no donation
|
||||
x_normed2, residual_out2 = ops.fused_add_rms_norm(
|
||||
x_normed1, residual2, self.weight2, 1e-5
|
||||
)
|
||||
return x_normed2, residual_out1, residual_out2
|
||||
|
||||
|
||||
class MixedModel(nn.Module):
|
||||
"""Model mixing maybe_inplace and functional variants."""
|
||||
|
||||
def __init__(self, hidden_size=16):
|
||||
super().__init__()
|
||||
self.weight1 = nn.Parameter(torch.ones(hidden_size, dtype=torch.bfloat16))
|
||||
self.weight2 = nn.Parameter(torch.ones(hidden_size, dtype=torch.bfloat16))
|
||||
|
||||
def forward(
|
||||
self, x: torch.Tensor, residual1: torch.Tensor, residual2: torch.Tensor
|
||||
):
|
||||
# First maybe_inplace - x & residual1 are donated
|
||||
x_normed1, residual_out1 = ops.fused_add_rms_norm.maybe_inplace(
|
||||
x, residual1, self.weight1, 1e-5
|
||||
)
|
||||
# Second functional - no donation, x_normed1 must be preserved as it's returned
|
||||
x_normed2, residual_out2 = ops.fused_add_rms_norm(
|
||||
x_normed1, residual2, self.weight2, 1e-5
|
||||
)
|
||||
# Return both to prevent x_normed1 from being optimized away
|
||||
return x_normed1, x_normed2, residual_out1, residual_out2
|
||||
|
||||
|
||||
class ModelWithTritonAfterMaybeInplace(nn.Module):
|
||||
"""
|
||||
Model using maybe_inplace followed by a Triton kernel.
|
||||
Test clone elimination can handle Triton in the graph
|
||||
"""
|
||||
|
||||
def __init__(self, hidden_size=16):
|
||||
super().__init__()
|
||||
self.weight = nn.Parameter(torch.ones(hidden_size, dtype=torch.bfloat16))
|
||||
|
||||
@triton.jit
|
||||
def _triton_add_kernel(
|
||||
x_ptr,
|
||||
y_ptr,
|
||||
n_elements,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
):
|
||||
pid = tl.program_id(axis=0)
|
||||
block_start = pid * BLOCK_SIZE
|
||||
offsets = block_start + tl.arange(0, BLOCK_SIZE)
|
||||
mask = offsets < n_elements
|
||||
x = tl.load(x_ptr + offsets, mask=mask)
|
||||
y = x + 0.1
|
||||
tl.store(y_ptr + offsets, y, mask=mask)
|
||||
|
||||
def triton_add(x: torch.Tensor) -> torch.Tensor:
|
||||
"""Simple Triton add kernel."""
|
||||
y = torch.empty_like(x)
|
||||
n_elements = x.numel()
|
||||
grid = (triton.cdiv(n_elements, 256),)
|
||||
_triton_add_kernel[grid](x, y, n_elements, BLOCK_SIZE=256)
|
||||
return y
|
||||
|
||||
self.triton_add = triton_add
|
||||
|
||||
def forward(self, x: torch.Tensor, residual: torch.Tensor, residual2: torch.Tensor):
|
||||
x_normed, residual_out = ops.fused_add_rms_norm.maybe_inplace(
|
||||
x, residual, self.weight, 1e-5
|
||||
)
|
||||
|
||||
x_processed = self.triton_add(x_normed)
|
||||
|
||||
# x_processed does not need to be cloned, residual2 does
|
||||
x_normed2, residual_out2 = ops.fused_add_rms_norm(
|
||||
x_processed, residual2, self.weight, 1e-5
|
||||
)
|
||||
return x_normed2, residual_out2
|
||||
|
||||
|
||||
skipif_no_triton = pytest.mark.skipif(not HAS_TRITON, reason="Requires Triton")
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda_alike(),
|
||||
reason="Only test on cuda and rocm platform",
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"model_class,expected_functionalized,expected_donated,expected_clones",
|
||||
[
|
||||
# 2 inplace calls, all activations donated, all clones eliminated
|
||||
(MaybeInplaceModel, 2, 3, 0),
|
||||
# No inplace calls, no donations, 3 clones (one eliminated)
|
||||
(FunctionalModel, 0, 0, 3),
|
||||
# One inplace call, two donated activations, 2 clones
|
||||
(MixedModel, 1, 2, 2),
|
||||
# One inplace call, two donated, 1 clone remaining
|
||||
pytest.param(ModelWithTritonAfterMaybeInplace, 1, 2, 1, marks=skipif_no_triton),
|
||||
],
|
||||
)
|
||||
def test_inplace_functionalization(
|
||||
default_vllm_config: VllmConfig,
|
||||
model_class,
|
||||
expected_functionalized: int,
|
||||
expected_clones: int,
|
||||
expected_donated: int,
|
||||
):
|
||||
"""Test inplace functionalization, lowering, and clone cleanup."""
|
||||
torch.set_default_device(current_platform.device_type)
|
||||
|
||||
# Use vllm_c so inplace path is triggered
|
||||
default_vllm_config.kernel_config.ir_op_priority.fused_add_rms_norm = [
|
||||
"vllm_c",
|
||||
"native",
|
||||
]
|
||||
|
||||
# Create passes in order they run during compilation
|
||||
functionalization_pass = VllmIRInplaceFunctionalizationPass(default_vllm_config)
|
||||
lowering_pass = VllmIRLoweringPass(default_vllm_config)
|
||||
donated_info_pass = StoreDonationInfoPass()
|
||||
cleanup_pass = UnsafeCloneEliminationPass(default_vllm_config)
|
||||
|
||||
# Set up backend with pre-grad pass
|
||||
backend = TestBackend(lowering_pass, donated_info_pass, cleanup_pass)
|
||||
backend.inductor_config["pre_grad_custom_pass"] = functionalization_pass
|
||||
|
||||
model = model_class()
|
||||
x = torch.randn(8, 16, dtype=torch.bfloat16)
|
||||
residual1 = torch.randn(8, 16, dtype=torch.bfloat16)
|
||||
residual2 = torch.randn(8, 16, dtype=torch.bfloat16)
|
||||
|
||||
with default_vllm_config.kernel_config.ir_op_priority.set_priority():
|
||||
# Reference output without optimization
|
||||
ref_output = model(x.clone(), residual1.clone(), residual2.clone())
|
||||
|
||||
# Compile with inplace optimization
|
||||
compiled_model = torch.compile(model, backend=backend, fullgraph=True)
|
||||
output = compiled_model(x.clone(), residual1.clone(), residual2.clone())
|
||||
|
||||
# Verify correctness (relaxed tolerance for bfloat16)
|
||||
for i in range(len(ref_output)):
|
||||
torch.testing.assert_close(output[i], ref_output[i], rtol=1e-2, atol=1e-2)
|
||||
|
||||
# Verify expected number of ops were functionalized
|
||||
func_ops = functionalization_pass.functionalized_ops
|
||||
assert len(func_ops) == int(bool(expected_functionalized))
|
||||
if expected_functionalized > 0:
|
||||
assert "fused_add_rms_norm" in func_ops
|
||||
assert func_ops["fused_add_rms_norm"] == expected_functionalized
|
||||
|
||||
# Verify lowering happened (2 ops in all cases)
|
||||
assert "fused_add_rms_norm" in lowering_pass.selected_impls
|
||||
assert len(lowering_pass.selected_impls["fused_add_rms_norm"]) == 2
|
||||
assert all(
|
||||
provider == "vllm_c"
|
||||
for node, provider in lowering_pass.selected_impls["fused_add_rms_norm"].items()
|
||||
), lowering_pass.selected_impls
|
||||
|
||||
# Verify correct number of donated IDs
|
||||
assert len(donated_info_pass.donated_input_ids_sets) == 1
|
||||
assert len(donated_info_pass.donated_input_ids_sets[0]) == expected_donated
|
||||
|
||||
# Verify expected number of clones after cleanup
|
||||
actual_clones = backend.op_count(torch.ops.aten.clone.default, before=False)
|
||||
assert actual_clones == expected_clones, (
|
||||
f"Expected {expected_clones} clones, got {actual_clones}:"
|
||||
f"{backend.print_graphs()}"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda_alike(),
|
||||
reason="Only test on cuda and rocm platform",
|
||||
)
|
||||
def test_donated_buffer_context_propagation(default_vllm_config):
|
||||
"""Test that donated_input_ids propagates correctly through pass_context."""
|
||||
torch.set_default_device(current_platform.device_type)
|
||||
|
||||
# Create a custom backend that inspects pass_context in cleanup pass
|
||||
functionalization_pass = VllmIRInplaceFunctionalizationPass(default_vllm_config)
|
||||
lowering_pass = VllmIRLoweringPass(default_vllm_config)
|
||||
|
||||
donation_info_pass = StoreDonationInfoPass()
|
||||
cleanup_pass = UnsafeCloneEliminationPass(default_vllm_config)
|
||||
|
||||
backend = TestBackend(lowering_pass, donation_info_pass, cleanup_pass)
|
||||
backend.inductor_config["pre_grad_custom_pass"] = functionalization_pass
|
||||
|
||||
model = MaybeInplaceModel()
|
||||
x = torch.randn(8, 16, dtype=torch.bfloat16)
|
||||
residual1 = torch.randn(8, 16, dtype=torch.bfloat16)
|
||||
residual2 = torch.randn(8, 16, dtype=torch.bfloat16)
|
||||
|
||||
compiled_model = torch.compile(model, backend=backend, fullgraph=True)
|
||||
compiled_model(x.clone(), residual1.clone(), residual2.clone())
|
||||
|
||||
donated_ids_seen = donation_info_pass.donated_input_ids_sets
|
||||
# Verify donated_input_ids was set and propagated
|
||||
assert len(donated_ids_seen) == 1
|
||||
# Should have donated inputs (exact indices depend on AOTAutograd)
|
||||
assert len(donated_ids_seen[0]) == 3
|
||||
# All donated ids should be valid non-negative integers
|
||||
for idx in donated_ids_seen[0]:
|
||||
assert isinstance(idx, int) and idx >= 0, f"Invalid donated index: {idx}"
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda_alike(),
|
||||
reason="Only test on cuda and rocm platform",
|
||||
)
|
||||
def test_maybe_inplace_reuse_error(default_vllm_config):
|
||||
"""Test that reusing a donated activation input raises ValueError."""
|
||||
torch.set_default_device(current_platform.device_type)
|
||||
|
||||
class ReuseModel(nn.Module):
|
||||
"""Model that incorrectly reuses a donated activation input."""
|
||||
|
||||
def __init__(self, hidden_size=16):
|
||||
super().__init__()
|
||||
self.weight = nn.Parameter(torch.ones(hidden_size, dtype=torch.bfloat16))
|
||||
|
||||
def forward(self, x: torch.Tensor, residual: torch.Tensor):
|
||||
# x is donated to maybe_inplace
|
||||
x_normed, residual_out = ops.fused_add_rms_norm.maybe_inplace(
|
||||
x, residual, self.weight, 1e-5
|
||||
)
|
||||
# ERROR: x is used again after being donated
|
||||
return x_normed + x # This should raise ValueError
|
||||
|
||||
functionalization_pass = VllmIRInplaceFunctionalizationPass(default_vllm_config)
|
||||
lowering_pass = VllmIRLoweringPass(default_vllm_config)
|
||||
cleanup_pass = UnsafeCloneEliminationPass(default_vllm_config)
|
||||
|
||||
backend = TestBackend(lowering_pass, cleanup_pass)
|
||||
backend.inductor_config["pre_grad_custom_pass"] = functionalization_pass
|
||||
|
||||
model = ReuseModel()
|
||||
x = torch.randn(8, 16, dtype=torch.bfloat16)
|
||||
residual = torch.randn(8, 16, dtype=torch.bfloat16)
|
||||
|
||||
# Compilation should raise BackendCompilerFailed wrapping ValueError
|
||||
with pytest.raises(
|
||||
torch._dynamo.exc.BackendCompilerFailed,
|
||||
match="is used again after the node",
|
||||
):
|
||||
compiled_model = torch.compile(model, backend=backend, fullgraph=True)
|
||||
compiled_model(x.clone(), residual.clone())
|
||||
|
||||
|
||||
# Piecewise compilation tests with graph splitting
|
||||
|
||||
|
||||
@torch.library.custom_op("vllm::test_split_marker", mutates_args=())
|
||||
def test_split_marker(x: torch.Tensor) -> torch.Tensor:
|
||||
"""Identity op that marks a split point for piecewise compilation."""
|
||||
return x.clone()
|
||||
|
||||
|
||||
@test_split_marker.register_fake
|
||||
def _fake_split_marker(x: torch.Tensor) -> torch.Tensor:
|
||||
return torch.empty_like(x)
|
||||
|
||||
|
||||
class TransformerBlockWithSplits(nn.Module):
|
||||
"""Transformer block with explicit split points for piecewise compilation."""
|
||||
|
||||
def __init__(self, hidden_size=32, intermediate_size=128):
|
||||
super().__init__()
|
||||
self.hidden_size = hidden_size
|
||||
self.intermediate_size = intermediate_size
|
||||
|
||||
# Attention-like projection
|
||||
self.attn_proj = nn.Linear(
|
||||
hidden_size, hidden_size, bias=False, dtype=torch.bfloat16
|
||||
)
|
||||
|
||||
# Post-attention norm
|
||||
self.post_attn_norm = nn.Parameter(
|
||||
torch.ones(hidden_size, dtype=torch.bfloat16)
|
||||
)
|
||||
|
||||
# MLP
|
||||
self.gate_proj = nn.Linear(
|
||||
hidden_size, intermediate_size, bias=False, dtype=torch.bfloat16
|
||||
)
|
||||
self.up_proj = nn.Linear(
|
||||
hidden_size, intermediate_size, bias=False, dtype=torch.bfloat16
|
||||
)
|
||||
self.down_proj = nn.Linear(
|
||||
intermediate_size, hidden_size, bias=False, dtype=torch.bfloat16
|
||||
)
|
||||
|
||||
# Post-MLP norm
|
||||
self.post_mlp_norm = nn.Parameter(torch.ones(hidden_size, dtype=torch.bfloat16))
|
||||
|
||||
def forward(self, x: torch.Tensor):
|
||||
# Attention block with residual
|
||||
residual1 = x
|
||||
attn_out = self.attn_proj(x)
|
||||
|
||||
# Fused add + norm (maybe_inplace: residual1 is donated)
|
||||
normed1, residual1 = ops.fused_add_rms_norm.maybe_inplace(
|
||||
attn_out, residual1, self.post_attn_norm, 1e-5
|
||||
)
|
||||
|
||||
# Force a graph split here
|
||||
normed1 = torch.ops.vllm.test_split_marker(normed1)
|
||||
|
||||
# MLP block
|
||||
gate = self.gate_proj(normed1)
|
||||
up = self.up_proj(normed1)
|
||||
mlp_out = self.down_proj(gate * torch.nn.functional.silu(up))
|
||||
|
||||
# Fused add + norm (maybe_inplace: residual1 is donated)
|
||||
normed2, residual2 = ops.fused_add_rms_norm.maybe_inplace(
|
||||
mlp_out, residual1, self.post_mlp_norm, 1e-5
|
||||
)
|
||||
|
||||
return normed2, residual2
|
||||
|
||||
|
||||
def with_dyn_arg(fn: Callable, arg_index: int, dim_index: int):
|
||||
def inner(*args):
|
||||
torch._dynamo.mark_dynamic(args[arg_index], dim_index)
|
||||
return fn(*args)
|
||||
|
||||
return inner
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda_alike(),
|
||||
reason="Only test on cuda and rocm platform",
|
||||
)
|
||||
def test_piecewise_compilation_with_donated_buffers(monkeypatch, fresh_vllm_cache):
|
||||
"""
|
||||
Test piecewise compilation with donated buffers across graph splits.
|
||||
Utilizes a custom splitting op. Uses fresh cache to avoid compilation caching.
|
||||
"""
|
||||
torch.set_default_device(current_platform.device_type)
|
||||
|
||||
# Disable compilation cache to avoid serialization issues
|
||||
monkeypatch.setenv("VLLM_DISABLE_COMPILE_CACHE", "1")
|
||||
|
||||
from vllm.compilation.backends import VllmBackend
|
||||
from vllm.config import CompilationConfig, VllmConfig
|
||||
|
||||
# Create config with custom splitting op
|
||||
store_donation_info = StoreDonationInfoPass()
|
||||
vllm_config = VllmConfig(
|
||||
compilation_config=CompilationConfig(
|
||||
custom_ops=["all"],
|
||||
splitting_ops=["vllm::test_split_marker"],
|
||||
inductor_compile_config={"post_grad_custom_post_pass": store_donation_info},
|
||||
)
|
||||
)
|
||||
|
||||
backend = VllmBackend(vllm_config)
|
||||
|
||||
model = TransformerBlockWithSplits()
|
||||
x = torch.randn(8, 32, dtype=torch.bfloat16)
|
||||
|
||||
# Reference output
|
||||
ref_output = with_dyn_arg(model, 0, 0)(x.clone())
|
||||
|
||||
# Compile with piecewise compilation (graph will split at split_marker)
|
||||
compiled_model = torch.compile(model, backend=backend, fullgraph=False)
|
||||
output = with_dyn_arg(compiled_model, 0, 0)(x.clone())
|
||||
|
||||
# Verify correctness (relaxed tolerance for bfloat16)
|
||||
torch.testing.assert_close(output[0], ref_output[0], rtol=1e-2, atol=1e-2)
|
||||
torch.testing.assert_close(output[1], ref_output[1], rtol=1e-2, atol=1e-2)
|
||||
|
||||
# Verify the model was split into multiple submodules
|
||||
assert hasattr(backend, "split_gm"), "Backend should have split graph module"
|
||||
|
||||
# Should have at least 2 submodules (split by test_split_marker op)
|
||||
submodules = list(backend.split_gm.named_children())
|
||||
num_submodules = len(submodules)
|
||||
assert num_submodules >= 2, (
|
||||
f"Expected at least 2 submodules (split), got {num_submodules}"
|
||||
)
|
||||
|
||||
# Check that donation info was propagated correctly
|
||||
donated_inputs_sets = store_donation_info.donated_input_ids_sets
|
||||
assert len(donated_inputs_sets) == 2
|
||||
assert len(donated_inputs_sets[0]) == 1
|
||||
assert len(donated_inputs_sets[1]) == 1
|
||||
@@ -126,7 +126,7 @@ class TestFusedAddRMSNorm(torch.nn.Module):
|
||||
if TEST_FP8 and do_fusion:
|
||||
return [torch.ops._C.fused_add_rms_norm_static_fp8_quant.default]
|
||||
else:
|
||||
return [torch.ops._C.fused_add_rms_norm.default]
|
||||
return []
|
||||
|
||||
def ops_not_in_model(self):
|
||||
return []
|
||||
|
||||
@@ -59,7 +59,7 @@ class TestModel(torch.nn.Module):
|
||||
|
||||
def ops_in_model_before(self):
|
||||
return [
|
||||
rocm_aiter_ops.get_rmsnorm_fused_add_op(),
|
||||
torch.ops.vllm_ir.fused_add_rms_norm,
|
||||
torch.ops.aten.constant_pad_nd,
|
||||
]
|
||||
|
||||
|
||||
@@ -17,7 +17,6 @@ from vllm.compilation.passes.fusion.rms_quant_fusion import (
|
||||
FusedRMSQuantKey,
|
||||
RMSNormQuantFusionPass,
|
||||
)
|
||||
from vllm.compilation.passes.fx_utils import find_op_nodes
|
||||
from vllm.compilation.passes.utility.noop_elimination import NoOpEliminationPass
|
||||
from vllm.compilation.passes.utility.post_cleanup import PostCleanupPass
|
||||
from vllm.config import (
|
||||
@@ -243,9 +242,10 @@ class TestModel(torch.nn.Module):
|
||||
]
|
||||
|
||||
def ops_in_model_before_partial(self):
|
||||
return [torch.ops.vllm_ir.rms_norm] + (
|
||||
[RMS_ADD_OP] if self.enable_rms_norm_custom_op else [torch.ops.aten.rsqrt]
|
||||
)
|
||||
return [
|
||||
torch.ops.vllm_ir.rms_norm,
|
||||
torch.ops.vllm_ir.fused_add_rms_norm.default,
|
||||
]
|
||||
|
||||
|
||||
def _run_fusion_test(
|
||||
@@ -383,17 +383,6 @@ def test_fusion_rmsnorm_quant(
|
||||
model.ops_in_model_before_partial(), fully_replaced=False
|
||||
)
|
||||
|
||||
# If RMSNorm custom op is disabled (native/torch impl used),
|
||||
# there's a risk that the fused add doesn't get included in the
|
||||
# replacement and only the rms part gets fused with quant.
|
||||
# Hence, we check only 2 add nodes are left (final fused rmsnorm add).
|
||||
if not enable_rms_norm_custom_op:
|
||||
n_add_nodes = lambda g: sum(1 for _ in find_op_nodes(torch.ops.aten.add, g))
|
||||
# rms_norm is IR, not included
|
||||
# 6 = 3x2 (3xRMS_ADD, 2 each)
|
||||
assert n_add_nodes(backend.graph_pre_pass) == 6
|
||||
assert n_add_nodes(backend.graph_post_pass) == 2
|
||||
|
||||
|
||||
@pytest.mark.parametrize("dtype", [torch.bfloat16])
|
||||
@pytest.mark.parametrize("hidden_size", [256])
|
||||
|
||||
Reference in New Issue
Block a user