[vLLM IR] 2/N fused_add_rms_norm and maybe_inplace overload (#36823)

Signed-off-by: Luka Govedič <lgovedic@redhat.com>
Signed-off-by: Luka Govedič <ProExpertProg@users.noreply.github.com>
This commit is contained in:
Luka Govedič
2026-05-01 23:41:15 -04:00
committed by GitHub
parent 3e49479c4b
commit d58c42e19c
44 changed files with 2833 additions and 779 deletions
+15 -2
View File
@@ -12,10 +12,17 @@ from torch._ops import OpOverload, OpOverloadPacket
from torch.fx._utils import lazy_format_graph_code
from vllm.compilation.passes.fx_utils import find_op_nodes
from vllm.compilation.passes.inductor_pass import InductorPass
from vllm.compilation.passes.inductor_pass import (
InductorPass,
pass_context,
)
from vllm.compilation.passes.ir.inplace_functionalization import (
VllmIRInplaceFunctionalizationPass,
)
from vllm.compilation.passes.pass_manager import with_pattern_match_debug
from vllm.compilation.passes.vllm_inductor_pass import VllmInductorPass
from vllm.config import VllmConfig, get_current_vllm_config
from vllm.config.utils import Range
from vllm.logger import init_logger
logger = init_logger("vllm.tests.compile.backend")
@@ -53,11 +60,17 @@ class TestBackend:
self.custom_passes = list(passes)
vllm_config = get_current_vllm_config()
compile_config = vllm_config.compilation_config
self.range = Range(1, vllm_config.scheduler_config.max_num_batched_tokens)
# Deepcopy to allow multiple TestBackend instances to use the same VllmConfig
self.inductor_config = deepcopy(compile_config.inductor_compile_config)
self.inductor_config["force_disable_caches"] = True
self.inductor_config["post_grad_custom_post_pass"] = self.post_pass
# Add VllmIRInplaceFunctionalizationPass as pre-grad pass by default
self.inductor_config["pre_grad_custom_pass"] = (
VllmIRInplaceFunctionalizationPass(vllm_config)
)
if debug_dump_path := vllm_config.compile_debug_dump_path():
logger.debug("Dumping depyf output to %s", debug_dump_path)
self.debug_ctx = depyf.prepare_debug(debug_dump_path.as_posix())
@@ -68,7 +81,7 @@ class TestBackend:
self.graph_pre_compile = deepcopy(graph)
from torch._inductor.compile_fx import compile_fx
with self.debug_ctx:
with self.debug_ctx, pass_context(self.range):
return compile_fx(
graph, example_inputs, config_patches=self.inductor_config
)
@@ -88,14 +88,10 @@ class TestAllReduceRMSNormModel(torch.nn.Module):
]
def ops_in_model(self):
return (
[torch.ops.vllm_ir.rms_norm]
+ [
torch.ops._C.fused_add_rms_norm.default,
]
if RMSNorm.enabled()
else []
)
return [
torch.ops.vllm_ir.rms_norm,
torch.ops.vllm_ir.fused_add_rms_norm,
]
class TestAllReduceRMSNormStaticQuantFP8Model(torch.nn.Module):
@@ -152,16 +148,17 @@ class TestAllReduceRMSNormStaticQuantFP8Model(torch.nn.Module):
def ops_in_model(self):
if self.vllm_config.compilation_config.pass_config.fuse_norm_quant:
return [torch.ops._C.fused_add_rms_norm_static_fp8_quant.default]
elif RMSNorm.enabled():
return [
torch.ops._C.fused_add_rms_norm.default,
]
elif any(layer.is_quant_fp8_enabled() for layer in self.fp8_linear_layers):
return [
torch.ops._C.static_scaled_fp8_quant.default,
]
else:
return []
quant_ops = (
[torch.ops._C.static_scaled_fp8_quant.default]
if any(layer.is_quant_fp8_enabled() for layer in self.fp8_linear_layers)
else [torch.ops.aten.reciprocal]
)
return [
torch.ops.vllm_ir.rms_norm,
torch.ops.vllm_ir.fused_add_rms_norm,
*quant_ops,
]
@multi_gpu_test(num_gpus=2)
@@ -0,0 +1,412 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Comprehensive tests for UnsafeCloneEliminationPass.
This test suite exercises all possible valid FX graph patterns involving clones:
1. Clone with no users (dead code)
2. Clone with read-only users
3. Clone with mutation users
4. Clone of graph input
5. Clone with original used after mutation
6. Clone chains
"""
import pytest
import torch
from torch import fx
from torch.fx.experimental.proxy_tensor import make_fx
from vllm.compilation.passes.fx_utils import find_op_nodes
from vllm.compilation.passes.inductor_pass import get_pass_context, pass_context
from vllm.compilation.passes.ir.clone_elimination import (
UnsafeCloneEliminationPass,
user_writes_to_node,
)
from vllm.config import VllmConfig
from vllm.config.utils import Range
def count_clones(graph: fx.Graph) -> int:
"""Count clone nodes in a graph."""
return len(list(find_op_nodes(torch.ops.aten.clone.default, graph)))
@pytest.fixture(scope="function")
def clone_cleanup_pass():
return UnsafeCloneEliminationPass(VllmConfig())
@pytest.fixture(autouse=True)
def setup_pass_context():
"""Set up pass context for each test."""
with pass_context(compile_range=Range(1, 8192)):
yield
class TestCloneCleanup:
"""Test UnsafeCloneEliminationPass behavior on various graph patterns."""
def test_remove_clone_readonly_users(self, clone_cleanup_pass):
"""Clone with only read-only users should be removed."""
def f(x: torch.Tensor) -> torch.Tensor:
x_clone = x.clone()
return x_clone + 1
inp = torch.randn(2, 3)
graph_module = make_fx(f)(inp)
assert count_clones(graph_module.graph) == 1
expected = graph_module(inp)
clone_cleanup_pass(graph_module.graph)
graph_module.recompile()
actual = graph_module(inp)
assert count_clones(graph_module.graph) == 0
torch.testing.assert_close(actual, expected)
def test_keep_clone_with_mutation_and_original_used_after(self, clone_cleanup_pass):
"""Clone must be kept if it's mutated AND original is used after mutation."""
def f(x: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
x = x.relu() # not a graph param
x_clone = x.clone()
x_clone.add_(1)
return x, x_clone
inp = torch.randn(2, 3)
graph_module = make_fx(f)(inp)
assert count_clones(graph_module.graph) == 1
expected = graph_module(inp)
clone_cleanup_pass(graph_module.graph)
graph_module.recompile()
actual = graph_module(inp)
# Clone should be KEPT because original is used after mutation
assert count_clones(graph_module.graph) == 1
torch.testing.assert_close(actual[0], expected[0])
torch.testing.assert_close(actual[1], expected[1])
def test_remove_clone_with_mutation_no_original_use(self, clone_cleanup_pass):
"""Clone can be removed if it's mutated but original is not used after."""
def f(x: torch.Tensor) -> torch.Tensor:
x = x.relu() # not a graph param
x_clone = x.clone()
x_clone.add_(1)
return x_clone
inp = torch.randn(2, 3)
graph_module = make_fx(f)(inp)
assert count_clones(graph_module.graph) == 1
expected = graph_module(inp)
clone_cleanup_pass(graph_module.graph)
graph_module.recompile()
actual = graph_module(inp)
assert count_clones(graph_module.graph) == 0
torch.testing.assert_close(actual, expected)
def test_clone_chain(self, clone_cleanup_pass):
"""Test handling of clone chains: x -> clone1 -> clone2."""
def f(x: torch.Tensor) -> torch.Tensor:
x = x.relu() # not a graph param
x1 = x.clone()
x2 = x1.clone()
return x2 + 1
inp = torch.randn(2, 3)
graph_module = make_fx(f)(inp)
assert count_clones(graph_module.graph) == 2
expected = graph_module(inp)
clone_cleanup_pass(graph_module.graph)
graph_module.recompile()
actual = graph_module(inp)
# Both clones should be removed
assert count_clones(graph_module.graph) == 0
torch.testing.assert_close(actual, expected)
def test_multiple_clones_of_same_input(self, clone_cleanup_pass):
"""Test multiple independent clones of the same input."""
def f(x: torch.Tensor) -> torch.Tensor:
x1 = x.clone()
x2 = x.clone()
return x1 + x2
inp = torch.randn(2, 3)
graph_module = make_fx(f)(inp)
assert count_clones(graph_module.graph) == 2
expected = graph_module(inp)
clone_cleanup_pass(graph_module.graph)
graph_module.recompile()
actual = graph_module(inp)
# Both clones should be removed (only readonly uses)
assert count_clones(graph_module.graph) == 0
torch.testing.assert_close(actual, expected)
def test_no_clones_in_graph(self, clone_cleanup_pass):
"""Test pass behavior when graph has no clones."""
def f(x: torch.Tensor) -> torch.Tensor:
return x + 1
inp = torch.randn(2, 3)
graph_module = make_fx(f)(inp)
assert count_clones(graph_module.graph) == 0
expected = graph_module(inp)
clone_cleanup_pass(graph_module.graph)
graph_module.recompile()
actual = graph_module(inp)
assert count_clones(graph_module.graph) == 0
torch.testing.assert_close(actual, expected)
def test_multiple_passes(self, clone_cleanup_pass):
"""Test running the pass multiple times (should be idempotent)."""
def f(x: torch.Tensor) -> torch.Tensor:
x1 = x.clone()
return x1 + 1
inp = torch.randn(2, 3)
graph_module = make_fx(f)(inp)
assert count_clones(graph_module.graph) == 1
expected = graph_module(inp)
clone_cleanup_pass(graph_module.graph)
assert count_clones(graph_module.graph) == 0
graph_module.recompile()
actual = graph_module(inp)
torch.testing.assert_close(actual, expected)
clone_cleanup_pass(graph_module.graph)
assert count_clones(graph_module.graph) == 0
graph_module.recompile()
actual = graph_module(inp)
torch.testing.assert_close(actual, expected)
def test_output_node_no_write(self):
"""Output nodes never write to their inputs."""
def f(x: torch.Tensor) -> torch.Tensor:
return x
graph_module = make_fx(f)(torch.randn(2, 3))
x_node = [n for n in graph_module.graph.nodes if n.op == "placeholder"][0]
output_node = [n for n in graph_module.graph.nodes if n.op == "output"][0]
assert not user_writes_to_node(output_node, x_node)
def test_readonly_op_no_write(self):
"""Readonly operations don't write to inputs."""
def f(x: torch.Tensor, y: torch.Tensor) -> torch.Tensor:
return x + y
graph_module = make_fx(f)(torch.randn(2, 3), torch.randn(2, 3))
placeholders = [n for n in graph_module.graph.nodes if n.op == "placeholder"]
add_node = [
n
for n in graph_module.graph.nodes
if n.op == "call_function" and n.target == torch.ops.aten.add.Tensor
][0]
assert not user_writes_to_node(add_node, placeholders[0])
assert not user_writes_to_node(add_node, placeholders[1])
def test_inplace_op_writes(self):
"""Inplace operations write to first argument."""
def f(x: torch.Tensor, y: torch.Tensor) -> torch.Tensor:
x.add_(y)
return x
graph_module = make_fx(f)(torch.randn(2, 3), torch.randn(2, 3))
placeholders = [n for n in graph_module.graph.nodes if n.op == "placeholder"]
add_node = [
n
for n in graph_module.graph.nodes
if n.op == "call_function" and "add_" in str(n.target)
][0]
# add_ writes to first arg but not second
assert user_writes_to_node(add_node, placeholders[0])
assert not user_writes_to_node(add_node, placeholders[1])
def test_copy_writes(self):
"""copy_ operation writes to first argument."""
def f(x: torch.Tensor, y: torch.Tensor) -> torch.Tensor:
x.copy_(y)
return x
graph_module = make_fx(f)(torch.randn(2, 3), torch.randn(2, 3))
placeholders = [n for n in graph_module.graph.nodes if n.op == "placeholder"]
copy_node = [
n
for n in graph_module.graph.nodes
if n.op == "call_function" and "copy_" in str(n.target)
][0]
assert user_writes_to_node(copy_node, placeholders[0])
assert not user_writes_to_node(copy_node, placeholders[1])
def test_auto_functionalized_not_a_write(self):
"""auto_functionalized ops are follow-up uses, not writes."""
from torch._higher_order_ops.auto_functionalize import auto_functionalized
def f(x: torch.Tensor) -> torch.Tensor:
return x
graph_module = make_fx(f)(torch.randn(2, 3))
x_node = [n for n in graph_module.graph.nodes if n.op == "placeholder"][0]
# Create an auto_functionalized node in the graph
with graph_module.graph.inserting_before(None):
af_node = graph_module.graph.call_function(
auto_functionalized, kwargs={"input": x_node}
)
# auto_functionalized should not be treated as a write
assert not user_writes_to_node(af_node, x_node)
def test_higher_order_op_conservatively_writes(self):
"""Other higher-order operators are conservatively treated as writes."""
from torch._ops import HigherOrderOperator
def f(x: torch.Tensor) -> torch.Tensor:
return x
graph_module = make_fx(f)(torch.randn(2, 3))
x_node = [n for n in graph_module.graph.nodes if n.op == "placeholder"][0]
# Create a concrete higher-order operator subclass
class MockHigherOrderOp(HigherOrderOperator):
def __call__(self, *args, **kwargs):
return args[0] if args else None
mock_hoo = MockHigherOrderOp("mock_higher_order_op")
with graph_module.graph.inserting_before(None):
hoo_node = graph_module.graph.call_function(mock_hoo, args=(x_node,))
# Should be conservative and assume it could write
assert user_writes_to_node(hoo_node, x_node)
class TestCloneCleanupWithDonatedInputs:
"""Test UnsafeCloneEliminationPass with donated input tracking via PassContext."""
@pytest.fixture(autouse=True)
def setup_pass_context(self):
"""Set up pass context for each test."""
with pass_context(compile_range=Range(1, 8192)):
yield
def test_donated_input_clone_removed(self, clone_cleanup_pass):
"""Clone of donated input should be removed."""
def f(x: torch.Tensor) -> torch.Tensor:
x_clone = x.clone()
x_clone.add_(1)
return x_clone
inp = torch.randn(2, 3)
graph_module = make_fx(f)(inp)
assert count_clones(graph_module.graph) == 1
# Mark first parameter as donated
get_pass_context().donated_input_ids = {0}
expected = graph_module(inp.clone())
clone_cleanup_pass(graph_module.graph)
graph_module.recompile()
# Clone should be removed since input is donated
assert count_clones(graph_module.graph) == 0
# Input can be mutated (donated)
inp_copy = inp.clone()
actual = graph_module(inp_copy)
torch.testing.assert_close(actual, expected)
def test_non_donated_input_clone_kept(self, clone_cleanup_pass):
"""Clone of non-donated input with mutation should be kept."""
def f(x: torch.Tensor, y: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
x_clone = x.clone()
x_clone.add_(1)
return x, x_clone
inp_x = torch.randn(2, 3)
inp_y = torch.randn(2, 3)
graph_module = make_fx(f)(inp_x, inp_y)
assert count_clones(graph_module.graph) == 1
# No donated inputs
get_pass_context().donated_input_ids = set()
expected = graph_module(inp_x.clone(), inp_y.clone())
clone_cleanup_pass(graph_module.graph)
graph_module.recompile()
# Clone should be kept since input is not donated and original is used
assert count_clones(graph_module.graph) == 1
# Verify inputs are not mutated
inp_x_before = inp_x.clone()
inp_y_before = inp_y.clone()
actual = graph_module(inp_x, inp_y)
torch.testing.assert_close(
inp_x, inp_x_before, msg="Input x should not be mutated"
)
torch.testing.assert_close(
inp_y, inp_y_before, msg="Input y should not be mutated"
)
torch.testing.assert_close(actual[0], expected[0])
torch.testing.assert_close(actual[1], expected[1])
def test_mixed_donated_inputs(self, clone_cleanup_pass):
"""Test with some inputs donated and some not."""
def f(x: torch.Tensor, y: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
x_clone = x.clone()
x_clone.add_(1)
y_clone = y.clone()
y_clone.add_(2)
return x_clone, y_clone
inp_x = torch.randn(2, 3)
inp_y = torch.randn(2, 3)
graph_module = make_fx(f)(inp_x, inp_y)
assert count_clones(graph_module.graph) == 2
# Only x is donated
get_pass_context().donated_input_ids = {0}
expected = graph_module(inp_x.clone(), inp_y.clone())
clone_cleanup_pass(graph_module.graph)
graph_module.recompile()
# x_clone removed (x is donated), y_clone kept (y is not donated)
assert count_clones(graph_module.graph) == 1
# Verify y is not mutated (x can be mutated since it's donated)
inp_y_before = inp_y.clone()
actual = graph_module(inp_x.clone(), inp_y)
torch.testing.assert_close(
inp_y, inp_y_before, msg="Input y should not be mutated"
)
torch.testing.assert_close(actual[0], expected[0])
torch.testing.assert_close(actual[1], expected[1])
@@ -0,0 +1,465 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Tests for IR inplace functionalization pass integration.
This test suite verifies that the inplace functionalization pass, lowering pass,
and clone cleanup pass work together correctly with donated buffer tracking.
"""
from collections.abc import Callable
import pytest
import torch
import torch._dynamo.exc
from torch import nn
import vllm.kernels # noqa: F401 to register kernels
from vllm.compilation.passes.inductor_pass import InductorPass, get_pass_context
from vllm.compilation.passes.ir.clone_elimination import (
UnsafeCloneEliminationPass,
)
from vllm.compilation.passes.ir.inplace_functionalization import (
VllmIRInplaceFunctionalizationPass,
)
from vllm.compilation.passes.ir.lowering_pass import VllmIRLoweringPass
from vllm.config import VllmConfig
from vllm.ir import ops
from vllm.platforms import current_platform
from vllm.triton_utils import HAS_TRITON, tl, triton
from ...backend import TestBackend
class StoreDonationInfoPass(InductorPass):
def __init__(self):
self.donated_input_ids_sets: list[set[int]] = []
def __call__(self, *args, **kwargs):
ctx = get_pass_context()
self.donated_input_ids_sets += [ctx.donated_input_ids]
class MaybeInplaceModel(nn.Module):
"""Model using only maybe_inplace variants."""
def __init__(self, hidden_size=16):
super().__init__()
self.weight1 = nn.Parameter(torch.ones(hidden_size, dtype=torch.bfloat16))
self.weight2 = nn.Parameter(torch.ones(hidden_size, dtype=torch.bfloat16))
def forward(
self, x: torch.Tensor, residual1: torch.Tensor, residual2: torch.Tensor
):
# First maybe_inplace - x & residual1 are donated
x_normed1, residual_out1 = ops.fused_add_rms_norm.maybe_inplace(
x, residual1, self.weight1, 1e-5
)
# Second maybe_inplace - residual2 is donated
x_normed2, residual_out2 = ops.fused_add_rms_norm.maybe_inplace(
x_normed1, residual2, self.weight2, 1e-5
)
return x_normed2, residual_out1, residual_out2
class FunctionalModel(nn.Module):
"""Model using only functional (default) variants."""
def __init__(self, hidden_size=16):
super().__init__()
self.weight1 = nn.Parameter(torch.ones(hidden_size, dtype=torch.bfloat16))
self.weight2 = nn.Parameter(torch.ones(hidden_size, dtype=torch.bfloat16))
def forward(
self, x: torch.Tensor, residual1: torch.Tensor, residual2: torch.Tensor
):
# First functional - no donation
x_normed1, residual_out1 = ops.fused_add_rms_norm(
x, residual1, self.weight1, 1e-5
)
# Second functional - no donation
x_normed2, residual_out2 = ops.fused_add_rms_norm(
x_normed1, residual2, self.weight2, 1e-5
)
return x_normed2, residual_out1, residual_out2
class MixedModel(nn.Module):
"""Model mixing maybe_inplace and functional variants."""
def __init__(self, hidden_size=16):
super().__init__()
self.weight1 = nn.Parameter(torch.ones(hidden_size, dtype=torch.bfloat16))
self.weight2 = nn.Parameter(torch.ones(hidden_size, dtype=torch.bfloat16))
def forward(
self, x: torch.Tensor, residual1: torch.Tensor, residual2: torch.Tensor
):
# First maybe_inplace - x & residual1 are donated
x_normed1, residual_out1 = ops.fused_add_rms_norm.maybe_inplace(
x, residual1, self.weight1, 1e-5
)
# Second functional - no donation, x_normed1 must be preserved as it's returned
x_normed2, residual_out2 = ops.fused_add_rms_norm(
x_normed1, residual2, self.weight2, 1e-5
)
# Return both to prevent x_normed1 from being optimized away
return x_normed1, x_normed2, residual_out1, residual_out2
class ModelWithTritonAfterMaybeInplace(nn.Module):
"""
Model using maybe_inplace followed by a Triton kernel.
Test clone elimination can handle Triton in the graph
"""
def __init__(self, hidden_size=16):
super().__init__()
self.weight = nn.Parameter(torch.ones(hidden_size, dtype=torch.bfloat16))
@triton.jit
def _triton_add_kernel(
x_ptr,
y_ptr,
n_elements,
BLOCK_SIZE: tl.constexpr,
):
pid = tl.program_id(axis=0)
block_start = pid * BLOCK_SIZE
offsets = block_start + tl.arange(0, BLOCK_SIZE)
mask = offsets < n_elements
x = tl.load(x_ptr + offsets, mask=mask)
y = x + 0.1
tl.store(y_ptr + offsets, y, mask=mask)
def triton_add(x: torch.Tensor) -> torch.Tensor:
"""Simple Triton add kernel."""
y = torch.empty_like(x)
n_elements = x.numel()
grid = (triton.cdiv(n_elements, 256),)
_triton_add_kernel[grid](x, y, n_elements, BLOCK_SIZE=256)
return y
self.triton_add = triton_add
def forward(self, x: torch.Tensor, residual: torch.Tensor, residual2: torch.Tensor):
x_normed, residual_out = ops.fused_add_rms_norm.maybe_inplace(
x, residual, self.weight, 1e-5
)
x_processed = self.triton_add(x_normed)
# x_processed does not need to be cloned, residual2 does
x_normed2, residual_out2 = ops.fused_add_rms_norm(
x_processed, residual2, self.weight, 1e-5
)
return x_normed2, residual_out2
skipif_no_triton = pytest.mark.skipif(not HAS_TRITON, reason="Requires Triton")
@pytest.mark.skipif(
not current_platform.is_cuda_alike(),
reason="Only test on cuda and rocm platform",
)
@pytest.mark.parametrize(
"model_class,expected_functionalized,expected_donated,expected_clones",
[
# 2 inplace calls, all activations donated, all clones eliminated
(MaybeInplaceModel, 2, 3, 0),
# No inplace calls, no donations, 3 clones (one eliminated)
(FunctionalModel, 0, 0, 3),
# One inplace call, two donated activations, 2 clones
(MixedModel, 1, 2, 2),
# One inplace call, two donated, 1 clone remaining
pytest.param(ModelWithTritonAfterMaybeInplace, 1, 2, 1, marks=skipif_no_triton),
],
)
def test_inplace_functionalization(
default_vllm_config: VllmConfig,
model_class,
expected_functionalized: int,
expected_clones: int,
expected_donated: int,
):
"""Test inplace functionalization, lowering, and clone cleanup."""
torch.set_default_device(current_platform.device_type)
# Use vllm_c so inplace path is triggered
default_vllm_config.kernel_config.ir_op_priority.fused_add_rms_norm = [
"vllm_c",
"native",
]
# Create passes in order they run during compilation
functionalization_pass = VllmIRInplaceFunctionalizationPass(default_vllm_config)
lowering_pass = VllmIRLoweringPass(default_vllm_config)
donated_info_pass = StoreDonationInfoPass()
cleanup_pass = UnsafeCloneEliminationPass(default_vllm_config)
# Set up backend with pre-grad pass
backend = TestBackend(lowering_pass, donated_info_pass, cleanup_pass)
backend.inductor_config["pre_grad_custom_pass"] = functionalization_pass
model = model_class()
x = torch.randn(8, 16, dtype=torch.bfloat16)
residual1 = torch.randn(8, 16, dtype=torch.bfloat16)
residual2 = torch.randn(8, 16, dtype=torch.bfloat16)
with default_vllm_config.kernel_config.ir_op_priority.set_priority():
# Reference output without optimization
ref_output = model(x.clone(), residual1.clone(), residual2.clone())
# Compile with inplace optimization
compiled_model = torch.compile(model, backend=backend, fullgraph=True)
output = compiled_model(x.clone(), residual1.clone(), residual2.clone())
# Verify correctness (relaxed tolerance for bfloat16)
for i in range(len(ref_output)):
torch.testing.assert_close(output[i], ref_output[i], rtol=1e-2, atol=1e-2)
# Verify expected number of ops were functionalized
func_ops = functionalization_pass.functionalized_ops
assert len(func_ops) == int(bool(expected_functionalized))
if expected_functionalized > 0:
assert "fused_add_rms_norm" in func_ops
assert func_ops["fused_add_rms_norm"] == expected_functionalized
# Verify lowering happened (2 ops in all cases)
assert "fused_add_rms_norm" in lowering_pass.selected_impls
assert len(lowering_pass.selected_impls["fused_add_rms_norm"]) == 2
assert all(
provider == "vllm_c"
for node, provider in lowering_pass.selected_impls["fused_add_rms_norm"].items()
), lowering_pass.selected_impls
# Verify correct number of donated IDs
assert len(donated_info_pass.donated_input_ids_sets) == 1
assert len(donated_info_pass.donated_input_ids_sets[0]) == expected_donated
# Verify expected number of clones after cleanup
actual_clones = backend.op_count(torch.ops.aten.clone.default, before=False)
assert actual_clones == expected_clones, (
f"Expected {expected_clones} clones, got {actual_clones}:"
f"{backend.print_graphs()}"
)
@pytest.mark.skipif(
not current_platform.is_cuda_alike(),
reason="Only test on cuda and rocm platform",
)
def test_donated_buffer_context_propagation(default_vllm_config):
"""Test that donated_input_ids propagates correctly through pass_context."""
torch.set_default_device(current_platform.device_type)
# Create a custom backend that inspects pass_context in cleanup pass
functionalization_pass = VllmIRInplaceFunctionalizationPass(default_vllm_config)
lowering_pass = VllmIRLoweringPass(default_vllm_config)
donation_info_pass = StoreDonationInfoPass()
cleanup_pass = UnsafeCloneEliminationPass(default_vllm_config)
backend = TestBackend(lowering_pass, donation_info_pass, cleanup_pass)
backend.inductor_config["pre_grad_custom_pass"] = functionalization_pass
model = MaybeInplaceModel()
x = torch.randn(8, 16, dtype=torch.bfloat16)
residual1 = torch.randn(8, 16, dtype=torch.bfloat16)
residual2 = torch.randn(8, 16, dtype=torch.bfloat16)
compiled_model = torch.compile(model, backend=backend, fullgraph=True)
compiled_model(x.clone(), residual1.clone(), residual2.clone())
donated_ids_seen = donation_info_pass.donated_input_ids_sets
# Verify donated_input_ids was set and propagated
assert len(donated_ids_seen) == 1
# Should have donated inputs (exact indices depend on AOTAutograd)
assert len(donated_ids_seen[0]) == 3
# All donated ids should be valid non-negative integers
for idx in donated_ids_seen[0]:
assert isinstance(idx, int) and idx >= 0, f"Invalid donated index: {idx}"
@pytest.mark.skipif(
not current_platform.is_cuda_alike(),
reason="Only test on cuda and rocm platform",
)
def test_maybe_inplace_reuse_error(default_vllm_config):
"""Test that reusing a donated activation input raises ValueError."""
torch.set_default_device(current_platform.device_type)
class ReuseModel(nn.Module):
"""Model that incorrectly reuses a donated activation input."""
def __init__(self, hidden_size=16):
super().__init__()
self.weight = nn.Parameter(torch.ones(hidden_size, dtype=torch.bfloat16))
def forward(self, x: torch.Tensor, residual: torch.Tensor):
# x is donated to maybe_inplace
x_normed, residual_out = ops.fused_add_rms_norm.maybe_inplace(
x, residual, self.weight, 1e-5
)
# ERROR: x is used again after being donated
return x_normed + x # This should raise ValueError
functionalization_pass = VllmIRInplaceFunctionalizationPass(default_vllm_config)
lowering_pass = VllmIRLoweringPass(default_vllm_config)
cleanup_pass = UnsafeCloneEliminationPass(default_vllm_config)
backend = TestBackend(lowering_pass, cleanup_pass)
backend.inductor_config["pre_grad_custom_pass"] = functionalization_pass
model = ReuseModel()
x = torch.randn(8, 16, dtype=torch.bfloat16)
residual = torch.randn(8, 16, dtype=torch.bfloat16)
# Compilation should raise BackendCompilerFailed wrapping ValueError
with pytest.raises(
torch._dynamo.exc.BackendCompilerFailed,
match="is used again after the node",
):
compiled_model = torch.compile(model, backend=backend, fullgraph=True)
compiled_model(x.clone(), residual.clone())
# Piecewise compilation tests with graph splitting
@torch.library.custom_op("vllm::test_split_marker", mutates_args=())
def test_split_marker(x: torch.Tensor) -> torch.Tensor:
"""Identity op that marks a split point for piecewise compilation."""
return x.clone()
@test_split_marker.register_fake
def _fake_split_marker(x: torch.Tensor) -> torch.Tensor:
return torch.empty_like(x)
class TransformerBlockWithSplits(nn.Module):
"""Transformer block with explicit split points for piecewise compilation."""
def __init__(self, hidden_size=32, intermediate_size=128):
super().__init__()
self.hidden_size = hidden_size
self.intermediate_size = intermediate_size
# Attention-like projection
self.attn_proj = nn.Linear(
hidden_size, hidden_size, bias=False, dtype=torch.bfloat16
)
# Post-attention norm
self.post_attn_norm = nn.Parameter(
torch.ones(hidden_size, dtype=torch.bfloat16)
)
# MLP
self.gate_proj = nn.Linear(
hidden_size, intermediate_size, bias=False, dtype=torch.bfloat16
)
self.up_proj = nn.Linear(
hidden_size, intermediate_size, bias=False, dtype=torch.bfloat16
)
self.down_proj = nn.Linear(
intermediate_size, hidden_size, bias=False, dtype=torch.bfloat16
)
# Post-MLP norm
self.post_mlp_norm = nn.Parameter(torch.ones(hidden_size, dtype=torch.bfloat16))
def forward(self, x: torch.Tensor):
# Attention block with residual
residual1 = x
attn_out = self.attn_proj(x)
# Fused add + norm (maybe_inplace: residual1 is donated)
normed1, residual1 = ops.fused_add_rms_norm.maybe_inplace(
attn_out, residual1, self.post_attn_norm, 1e-5
)
# Force a graph split here
normed1 = torch.ops.vllm.test_split_marker(normed1)
# MLP block
gate = self.gate_proj(normed1)
up = self.up_proj(normed1)
mlp_out = self.down_proj(gate * torch.nn.functional.silu(up))
# Fused add + norm (maybe_inplace: residual1 is donated)
normed2, residual2 = ops.fused_add_rms_norm.maybe_inplace(
mlp_out, residual1, self.post_mlp_norm, 1e-5
)
return normed2, residual2
def with_dyn_arg(fn: Callable, arg_index: int, dim_index: int):
def inner(*args):
torch._dynamo.mark_dynamic(args[arg_index], dim_index)
return fn(*args)
return inner
@pytest.mark.skipif(
not current_platform.is_cuda_alike(),
reason="Only test on cuda and rocm platform",
)
def test_piecewise_compilation_with_donated_buffers(monkeypatch, fresh_vllm_cache):
"""
Test piecewise compilation with donated buffers across graph splits.
Utilizes a custom splitting op. Uses fresh cache to avoid compilation caching.
"""
torch.set_default_device(current_platform.device_type)
# Disable compilation cache to avoid serialization issues
monkeypatch.setenv("VLLM_DISABLE_COMPILE_CACHE", "1")
from vllm.compilation.backends import VllmBackend
from vllm.config import CompilationConfig, VllmConfig
# Create config with custom splitting op
store_donation_info = StoreDonationInfoPass()
vllm_config = VllmConfig(
compilation_config=CompilationConfig(
custom_ops=["all"],
splitting_ops=["vllm::test_split_marker"],
inductor_compile_config={"post_grad_custom_post_pass": store_donation_info},
)
)
backend = VllmBackend(vllm_config)
model = TransformerBlockWithSplits()
x = torch.randn(8, 32, dtype=torch.bfloat16)
# Reference output
ref_output = with_dyn_arg(model, 0, 0)(x.clone())
# Compile with piecewise compilation (graph will split at split_marker)
compiled_model = torch.compile(model, backend=backend, fullgraph=False)
output = with_dyn_arg(compiled_model, 0, 0)(x.clone())
# Verify correctness (relaxed tolerance for bfloat16)
torch.testing.assert_close(output[0], ref_output[0], rtol=1e-2, atol=1e-2)
torch.testing.assert_close(output[1], ref_output[1], rtol=1e-2, atol=1e-2)
# Verify the model was split into multiple submodules
assert hasattr(backend, "split_gm"), "Backend should have split graph module"
# Should have at least 2 submodules (split by test_split_marker op)
submodules = list(backend.split_gm.named_children())
num_submodules = len(submodules)
assert num_submodules >= 2, (
f"Expected at least 2 submodules (split), got {num_submodules}"
)
# Check that donation info was propagated correctly
donated_inputs_sets = store_donation_info.donated_input_ids_sets
assert len(donated_inputs_sets) == 2
assert len(donated_inputs_sets[0]) == 1
assert len(donated_inputs_sets[1]) == 1
@@ -126,7 +126,7 @@ class TestFusedAddRMSNorm(torch.nn.Module):
if TEST_FP8 and do_fusion:
return [torch.ops._C.fused_add_rms_norm_static_fp8_quant.default]
else:
return [torch.ops._C.fused_add_rms_norm.default]
return []
def ops_not_in_model(self):
return []
@@ -59,7 +59,7 @@ class TestModel(torch.nn.Module):
def ops_in_model_before(self):
return [
rocm_aiter_ops.get_rmsnorm_fused_add_op(),
torch.ops.vllm_ir.fused_add_rms_norm,
torch.ops.aten.constant_pad_nd,
]
+4 -15
View File
@@ -17,7 +17,6 @@ from vllm.compilation.passes.fusion.rms_quant_fusion import (
FusedRMSQuantKey,
RMSNormQuantFusionPass,
)
from vllm.compilation.passes.fx_utils import find_op_nodes
from vllm.compilation.passes.utility.noop_elimination import NoOpEliminationPass
from vllm.compilation.passes.utility.post_cleanup import PostCleanupPass
from vllm.config import (
@@ -243,9 +242,10 @@ class TestModel(torch.nn.Module):
]
def ops_in_model_before_partial(self):
return [torch.ops.vllm_ir.rms_norm] + (
[RMS_ADD_OP] if self.enable_rms_norm_custom_op else [torch.ops.aten.rsqrt]
)
return [
torch.ops.vllm_ir.rms_norm,
torch.ops.vllm_ir.fused_add_rms_norm.default,
]
def _run_fusion_test(
@@ -383,17 +383,6 @@ def test_fusion_rmsnorm_quant(
model.ops_in_model_before_partial(), fully_replaced=False
)
# If RMSNorm custom op is disabled (native/torch impl used),
# there's a risk that the fused add doesn't get included in the
# replacement and only the rms part gets fused with quant.
# Hence, we check only 2 add nodes are left (final fused rmsnorm add).
if not enable_rms_norm_custom_op:
n_add_nodes = lambda g: sum(1 for _ in find_op_nodes(torch.ops.aten.add, g))
# rms_norm is IR, not included
# 6 = 3x2 (3xRMS_ADD, 2 each)
assert n_add_nodes(backend.graph_pre_pass) == 6
assert n_add_nodes(backend.graph_post_pass) == 2
@pytest.mark.parametrize("dtype", [torch.bfloat16])
@pytest.mark.parametrize("hidden_size", [256])
+91
View File
@@ -0,0 +1,91 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
from torch import Tensor
from torch.fx.experimental.proxy_tensor import make_fx
import vllm.ir.op
from vllm.ir.op import IrOp, IrOpInplaceOverload
@vllm.ir.register_op(allow_inplace=True)
def _custom_mm2(x: Tensor, w: Tensor) -> Tensor:
return x @ w
@_custom_mm2.register_impl("regular")
def _custom_mm2_regular(x: Tensor, w: Tensor) -> Tensor:
return x @ w + 1
@_custom_mm2.register_impl("inplace", inplace=True)
def _custom_mm2_inplace(x: Tensor, w: Tensor) -> Tensor:
x.copy_(x @ w + 2)
return x
class TestInplaceOp:
def test_registration(self):
# Test that the inplace op is registered correctly.
assert "_custom_mm2" in IrOp.registry
assert IrOp.registry["_custom_mm2"] is _custom_mm2
assert _custom_mm2.torch_op is torch.ops.vllm_ir._custom_mm2.default
assert isinstance(_custom_mm2.maybe_inplace, IrOpInplaceOverload)
assert (
_custom_mm2.maybe_inplace.torch_op
is torch.ops.vllm_ir._custom_mm2.maybe_inplace
)
def test_inplace_dispatching(self):
# check that the correct implementation is dispatched based on priority,
# and inplace semantics hold
w = torch.randn(3, 3)
x = torch.randn(2, 3)
x1 = x.clone()
with _custom_mm2.set_priority(["regular"]):
result_regular = _custom_mm2.maybe_inplace(x, w)
# check that the regular op does not modify x
torch.testing.assert_close(x, x1, atol=0, rtol=0)
with _custom_mm2.set_priority(["inplace"]):
result_inplace: Tensor = _custom_mm2.maybe_inplace(x, w)
# check that the inplace op returns x directly
assert result_inplace.data_ptr() == x.data_ptr()
torch.testing.assert_close(result_inplace, x1 @ w + 2)
torch.testing.assert_close(result_regular, x1 @ w + 1)
def test_default_dispatching(self):
# check that the correct implementation is dispatched,
# and ops do not modify inputs when using the default overload
w = torch.randn(3, 3)
x = torch.randn(2, 3)
x1 = x.clone()
with _custom_mm2.set_priority(["regular"]):
result_regular = _custom_mm2(x, w)
with _custom_mm2.set_priority(["inplace"]):
result_inplace = _custom_mm2(x, w)
# check that x was not modified by either impl
torch.testing.assert_close(x, x1, atol=0, rtol=0)
torch.testing.assert_close(result_inplace, x1 @ w + 2)
torch.testing.assert_close(result_regular, x1 @ w + 1)
def test_trace(self):
# Test that the inplace op can be used in a graph.
def func(x: Tensor, y: Tensor) -> Tensor:
return _custom_mm2.maybe_inplace(x, y)
x = torch.randn(2, 3)
y = torch.randn(3, 4)
graph = make_fx(func)(x, y)
assert any(
node.target == torch.ops.vllm_ir._custom_mm2.maybe_inplace
for node in graph.graph.nodes
)
+27 -16
View File
@@ -21,7 +21,7 @@ class CustomError(Exception):
pass
@vllm.ir.register_op
@vllm.ir.register_op(allow_inplace=True)
def _custom_add(x: torch.Tensor, y: torch.Tensor) -> torch.Tensor:
return x + y
@@ -129,11 +129,15 @@ class TestIrOpCustomAdd:
@pytest.mark.parametrize("enable_torch_wrap", [True, False])
@pytest.mark.parametrize("symbolic_trace", [True, False])
@pytest.mark.parametrize("overload", ["default", "maybe_inplace"])
def test_trace_sees_single_custom_op(
self, symbolic_trace: bool, enable_torch_wrap: bool
self, symbolic_trace: bool, enable_torch_wrap: bool, overload: str
):
op_fn = _custom_add if overload == "default" else _custom_add.maybe_inplace
torch_op = getattr(torch.ops.vllm_ir._custom_add, overload)
def fn(x, y):
return _custom_add(x, y)
return op_fn(x, y)
def find_fn(target: Any, gm: fx.GraphModule):
return gm.graph.find_nodes(op="call_function", target=target)
@@ -155,7 +159,7 @@ class TestIrOpCustomAdd:
torch.testing.assert_close(out_fx, out_eager)
# check that IR nodes only appear if enable_torch_wrap=True
ir_nodes = find_fn(torch.ops.vllm_ir._custom_add.default, gm)
ir_nodes = find_fn(torch_op, gm)
if enable_torch_wrap:
assert len(ir_nodes) == 1, gm.code
else:
@@ -167,7 +171,7 @@ class TestIrOpCustomAdd:
else:
gm = make_fx(fn)(torch.randn(2, 2), torch.randn(2, 2))
ir_nodes = find_fn(torch.ops.vllm_ir._custom_add.default, gm)
ir_nodes = find_fn(torch_op, gm)
assert len(ir_nodes) == 1, gm.code
@@ -176,9 +180,12 @@ def impl_a(x: torch.Tensor, y: torch.Tensor) -> torch.Tensor:
return x + y + 10
@_custom_add.register_impl("impl_b")
@_custom_add.register_impl("impl_b", inplace=True)
def impl_b(x: torch.Tensor, y: torch.Tensor) -> torch.Tensor:
return x + y + 20
"""Computes x+y+20"""
x.add_(y)
x.add_(20)
return x
@_custom_add.register_impl("impl_even", supports_args=lambda x, y: x.size(1) % 2 == 0)
@@ -243,19 +250,23 @@ class TestIrOpImplDispatch:
# Restored to empty
assert _custom_add.get_priority() == []
def test_dispatch_priority_order(self):
@pytest.mark.parametrize("overload", ["default", "maybe_inplace"])
def test_dispatch_priority_order(self, overload: str):
op_fn = _custom_add if overload == "default" else _custom_add.maybe_inplace
torch_op = getattr(torch.ops.vllm_ir._custom_add, overload)
x = torch.tensor(1, dtype=torch.int32)
y = torch.tensor(2, dtype=torch.int32)
with _custom_add.set_priority(["impl_b", "impl_a"]):
assert _custom_add.dispatch(x, y) is impl_b
out1 = _custom_add(x, y)
out2 = torch.ops.vllm_ir._custom_add(x, y)
out1 = op_fn(x.clone(), y)
out2 = torch_op(x.clone(), y)
with _custom_add.set_priority(["impl_a"]):
assert _custom_add.dispatch(x, y) is impl_a
out3 = _custom_add(x, y)
out4 = torch.ops.vllm_ir._custom_add(x, y)
out3 = op_fn(x.clone(), y)
out4 = torch_op(x.clone(), y)
# impl_b
assert out1.item() == 1 + 2 + 20
@@ -265,18 +276,18 @@ class TestIrOpImplDispatch:
assert out4.item() == 1 + 2 + 10
def test_unsupported_impl_filtered(self):
@_custom_add.register_impl("unsupported", supported=False)
def impl_bad(x: torch.Tensor, y: torch.Tensor) -> torch.Tensor:
@_custom_add.register_impl("impl_unsupported", supported=False)
def impl_unsupported(x: torch.Tensor, y: torch.Tensor) -> torch.Tensor:
return x + y + 999
x = torch.tensor(1, dtype=torch.int32)
y = torch.tensor(2, dtype=torch.int32)
with _custom_add.set_priority(["unsupported", "impl_a"]):
with _custom_add.set_priority(["impl_unsupported", "impl_a"]):
assert _custom_add.get_priority() == ["impl_a"]
out = _custom_add(x, y)
# impl_bad skipped → impl_a
# impl_unsupported skipped → impl_a
assert out.item() == 1 + 2 + 10
def test_supports_args_runtime_dispatch_and_warning(
+205 -1
View File
@@ -28,7 +28,9 @@ def test_rms_norm_registration():
"native": True,
"vllm_c": current_platform.is_cuda_alike(),
"aiter": current_platform.is_rocm(),
"oink": False,
"oink": current_platform.has_device_capability(100)
and hasattr(torch.ops, "oink")
and hasattr(torch.ops.oink, "rmsnorm"),
"xpu_kernels": current_platform.is_xpu(),
}
@@ -67,6 +69,14 @@ class TestRMSNorm:
out2 = rms_norm_native(x * 2.0, weight, epsilon=epsilon)
torch.testing.assert_close(out2, out, rtol=get_default_rtol(out), atol=1e-3)
# Mean square should be approximately 1 (ignoring epsilon and weight scaling)
combined_norm = out.float() / weight.float()
variance = combined_norm.pow(2).mean(dim=-1)
# After RMS normalization, variance should be close to 1
torch.testing.assert_close(
variance, torch.ones_like(variance), rtol=1e-2, atol=1e-2
)
# Check behavior with and without weight
weight1 = torch.ones_like(weight)
out3 = rms_norm_native(x, weight1, epsilon=epsilon)
@@ -129,3 +139,197 @@ def test_aiter_rejects_unsupported_dtypes():
num_tokens=8, hidden_size=4096, dtype=dtype, epsilon=1e-5
)
assert not impl.supports_args(*args), f"aiter should reject dtype={dtype}"
fused_add_rms_norm_native = ir.ops.fused_add_rms_norm.impls["native"].impl_fn
@pytest.mark.skipif(
not current_platform.is_cuda_alike() and not current_platform.is_xpu(),
reason="Currently only kernels on CUDA, ROCm and XPU",
)
def test_fused_add_rms_norm_registration():
expected = {
"native": True,
"vllm_c": current_platform.is_cuda_alike(),
"aiter": current_platform.is_rocm(),
"oink": current_platform.has_device_capability(100)
and hasattr(torch.ops, "oink")
and hasattr(torch.ops.oink, "fused_add_rms_norm"),
"xpu_kernels": current_platform.is_xpu(),
}
actual = {
provider: impl.supported
for provider, impl in ir.ops.fused_add_rms_norm.impls.items()
}
assert actual == expected
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16, torch.float32])
@pytest.mark.parametrize("n_tokens", NUM_TOKENS)
@pytest.mark.parametrize("hidden_size", COMMON_HIDDEN_SIZES)
@pytest.mark.parametrize("epsilon", [1e-6, 1e-5])
@pytest.mark.skipif(
not current_platform.is_cuda_alike() and not current_platform.is_xpu(),
reason="Currently only kernels on CUDA, ROCm and XPU",
)
class TestFusedAddRMSNorm:
@classmethod
def setup_class(cls, **kwargs):
torch.set_default_device(current_platform.device_type)
def test_native_semantics(self, dtype, n_tokens, hidden_size, epsilon):
x, x_residual, weight, eps = ir.ops.fused_add_rms_norm.generate_inputs(
num_tokens=4, hidden_size=8, dtype=dtype, epsilon=epsilon
)
out, residual_out = fused_add_rms_norm_native(x, x_residual, weight, eps)
# Check shape, dtype, device
assert out.shape == x.shape
assert out.dtype == x.dtype
assert out.device == x.device
assert residual_out.shape == x_residual.shape
assert residual_out.dtype == x_residual.dtype
assert residual_out.device == x_residual.device
# Check that residual_out = x + x_residual
expected_residual = (x.float() + x_residual.float()).to(dtype)
torch.testing.assert_close(
residual_out, expected_residual, rtol=1e-3, atol=1e-3
)
# Verify that the output is RMS normalized version of (x + x_residual)
expected_out = rms_norm_native(expected_residual, weight, epsilon)
assert_close(
ir.ops.fused_add_rms_norm,
(out, residual_out),
(expected_out, expected_residual),
)
# Check the scaling property of rms norm
out1, _ = fused_add_rms_norm_native(
x, torch.zeros_like(x), weight, epsilon=epsilon
)
out2, _ = fused_add_rms_norm_native(
x * 2.0, torch.zeros_like(x), weight, epsilon=epsilon
)
torch.testing.assert_close(out2, out1, rtol=get_default_rtol(out), atol=1e-3)
# Check behavior with and without weight
weight1 = torch.ones_like(weight)
out3, _ = fused_add_rms_norm_native(x, x_residual, weight1, eps)
out4, _ = fused_add_rms_norm_native(x, x_residual, None, eps)
torch.testing.assert_close(out3, out4)
@pytest.mark.parametrize("provider", supported_providers(ir.ops.fused_add_rms_norm))
def test_impls(self, dtype, n_tokens, hidden_size, epsilon, provider):
impl = ir.ops.fused_add_rms_norm.impls[provider]
x, x_residual, weight, eps = ir.ops.fused_add_rms_norm.generate_inputs(
num_tokens=n_tokens, hidden_size=hidden_size, dtype=dtype, epsilon=epsilon
)
args = (x, x_residual, weight, eps, None)
if not impl.supports_args(*args):
pytest.skip(f"{provider} does not support args")
ref_output, ref_residual = fused_add_rms_norm_native(*clone_args(args))
output, residual = impl.impl_fn(*clone_args(args))
assert_close(ir.ops.fused_add_rms_norm, output, ref_output)
assert_close(ir.ops.fused_add_rms_norm, residual, ref_residual)
# check that dispatched call matches direct call
with ir.ops.fused_add_rms_norm.set_priority([provider, "native"]):
out_dispatched, residual_dispatched = ir.ops.fused_add_rms_norm(*args[:4])
out_direct, residual_direct = impl.impl_fn(*clone_args(args))
torch.testing.assert_close(out_dispatched, out_direct, rtol=0.0, atol=0.0)
torch.testing.assert_close(
residual_dispatched, residual_direct, rtol=0.0, atol=0.0
)
# none of these support variance_size override
assert not impl.supports_args(x, x_residual, weight, epsilon, 4)
assert not impl.supports_args(x, x_residual, weight, epsilon, variance_size=4)
# test weight=None behavior
out_no_weight, residual_no_weight = impl.impl_fn(
x.clone(), x_residual.clone(), None, epsilon
)
out_unit_weight, residual_unit_weight = impl.impl_fn(
x.clone(), x_residual.clone(), torch.ones_like(weight), epsilon
)
assert_close(ir.ops.fused_add_rms_norm, out_no_weight, out_unit_weight)
assert_close(
ir.ops.fused_add_rms_norm, residual_no_weight, residual_unit_weight
)
@pytest.mark.parametrize("provider", ["vllm_c"])
def test_inplace_semantics(self, dtype, n_tokens, hidden_size, epsilon, provider):
"""Test that inplace implementations reuse inputs,
for maybe_inplace overload but not for default overload."""
impl = ir.ops.fused_add_rms_norm.impls[provider]
if not impl.supported:
pytest.skip(f"{provider} impl not supported on this platform")
x, x_residual, weight, eps = ir.ops.fused_add_rms_norm.generate_inputs(
num_tokens=n_tokens, hidden_size=hidden_size, dtype=dtype, epsilon=epsilon
)
# Test default overload - should NOT modify inputs even with inplace impl
x_default = x.clone()
x_residual_default = x_residual.clone()
x_default_ptr = x_default.data_ptr()
x_residual_default_ptr = x_residual_default.data_ptr()
with ir.ops.fused_add_rms_norm.set_priority([provider, "native"]):
out_default, residual_default = ir.ops.fused_add_rms_norm(
x_default, x_residual_default, weight, eps
)
# Default should NOT be inplace (even with inplace implementation)
assert out_default.data_ptr() != x_default_ptr
assert residual_default.data_ptr() != x_residual_default_ptr
torch.testing.assert_close(x, x_default, rtol=0.0, atol=0.0)
torch.testing.assert_close(x_residual, x_residual_default, rtol=0.0, atol=0.0)
# Test maybe_inplace overload - should modify inputs with inplace impl
x_inplace = x.clone()
x_residual_inplace = x_residual.clone()
x_inplace_ptr = x_inplace.data_ptr()
x_residual_inplace_ptr = x_residual_inplace.data_ptr()
with ir.ops.fused_add_rms_norm.set_priority([provider, "native"]):
out_inplace, residual_inplace = ir.ops.fused_add_rms_norm.maybe_inplace(
x_inplace, x_residual_inplace, weight, eps
)
# maybe_inplace should be inplace
assert out_inplace.data_ptr() == x_inplace_ptr
assert residual_inplace.data_ptr() == x_residual_inplace_ptr
# Both should produce same results
torch.testing.assert_close(out_default, out_inplace, atol=0.0, rtol=0.0)
torch.testing.assert_close(
residual_default, residual_inplace, atol=0.0, rtol=0.0
)
@pytest.mark.parametrize("provider", supported_providers(ir.ops.fused_add_rms_norm))
def test_torch_opcheck(self, dtype, n_tokens, hidden_size, epsilon, provider):
args = ir.ops.fused_add_rms_norm.generate_inputs(
num_tokens=n_tokens, hidden_size=hidden_size, dtype=dtype, epsilon=epsilon
)
args = args + (None,) # Add variance_size parameter
# When checking the torch op, we have to set priority and use dispatch
with ir.ops.fused_add_rms_norm.set_priority([provider, "native"]):
torch.library.opcheck(torch.ops.vllm_ir.fused_add_rms_norm.default, args)
# Only test maybe_inplace with non-inplace implementations
# Inplace implementations return aliases of inputs which is not allowed.
# We break this invariant, but we also convert maybe_inplace to the default
# overload during compilation, so maybe_inplace never reaches Inductor.
if not ir.ops.fused_add_rms_norm.impls[provider].inplace:
torch.library.opcheck(
torch.ops.vllm_ir.fused_add_rms_norm.maybe_inplace, args
)
@@ -23,11 +23,7 @@ from vllm.model_executor.layers.fused_moe.router.fused_topk_router import (
vllm_topk_sigmoid,
vllm_topk_softmax,
)
from vllm.model_executor.layers.layernorm import (
RMSNorm,
dispatch_rocm_rmsnorm_func,
fused_add_rms_norm,
)
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.platforms import current_platform
RMS_NORM_SUPPORTED_DTYPES = [torch.float16, torch.bfloat16]
@@ -153,26 +149,3 @@ def test_topk_sigmoid_dispatch(use_rocm_aiter: bool):
assert topk_func == rocm_aiter_ops.topk_sigmoid
else:
assert topk_func == vllm_topk_sigmoid
@pytest.mark.parametrize("add_residual", [False])
@pytest.mark.parametrize("dtype", [torch.float32, torch.float16, torch.bfloat16])
@pytest.mark.parametrize("use_rocm_aiter", [True, False])
@pytest.mark.skipif(
not current_platform.is_rocm(), reason="AITER is a feature exclusive for ROCm"
)
def test_rms_norm_dispatch(
add_residual: bool, dtype: torch.dtype, use_rocm_aiter: bool
):
rms_norm_func = dispatch_rocm_rmsnorm_func(dtype, use_rocm_aiter)
should_use_rocm_aiter = (
current_platform.is_rocm()
and use_rocm_aiter
and dtype in RMS_NORM_SUPPORTED_DTYPES
)
if should_use_rocm_aiter:
assert rms_norm_func == rocm_aiter_ops.rms_norm2d_with_add
else:
assert rms_norm_func == fused_add_rms_norm
+78 -41
View File
@@ -1,60 +1,97 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import multiprocessing
import types
import pytest
import torch
from vllm.platforms import current_platform
def _load_oink_ops_module():
# Import the module normally (vllm is installed as an editable package in CI).
from vllm import _oink_ops
def _test_oink_availability_impl(
device_capability: tuple[int, int],
has_rmsnorm: bool,
has_fused_add_rms_norm: bool,
expected_available: bool,
expected_fused: bool,
) -> None:
"""Test OINK support detection with mocked state."""
import torch
return _oink_ops
from vllm import platforms
# Mock device capability (class method, override on class)
dc = platforms.interface.DeviceCapability(*device_capability)
platforms.current_platform.__class__.get_device_capability = lambda device_id=0: dc
# Mock oink ops
oink_ops = types.SimpleNamespace()
if has_rmsnorm:
oink_ops.rmsnorm = lambda x, w, eps: x
if has_fused_add_rms_norm:
oink_ops.fused_add_rms_norm = lambda x, residual, w, eps: None
torch.ops.oink = oink_ops
# Now import vllm modules with mocks in place (fresh import with mocked platform)
import vllm.kernels.oink_ops # noqa: F401
from vllm.ir.ops import fused_add_rms_norm, rms_norm
# Verify support checks
assert rms_norm.impls["oink"].supported is expected_available
assert fused_add_rms_norm.impls["oink"].supported is expected_fused
def test_oink_availability_checks(monkeypatch: pytest.MonkeyPatch):
_oink_ops = _load_oink_ops_module()
@pytest.mark.parametrize(
"device_capability,has_rmsnorm,has_fused_add_rms_norm,expected_available,expected_fused",
[
# Case 1: < SM100, ops not supported
((9, 0), True, False, False, False),
# Case 2: CUDA available and SM100, rmsnorm op registered
((10, 0), True, False, True, False),
# Case 3: SM100 with both rmsnorm and fused_add_rms_norm
((10, 0), True, True, True, True),
],
)
@pytest.mark.skipif(not current_platform.is_cuda(), reason="Only test on CUDA")
def test_oink_availability_checks(
device_capability: tuple[int, int],
has_rmsnorm: bool,
has_fused_add_rms_norm: bool,
expected_available: bool,
expected_fused: bool,
):
"""Test OINK support detection with clean import state for each parameter set."""
# Ensure the ops namespace exists and is mutable for tests.
monkeypatch.setattr(
torch.ops,
"oink",
types.SimpleNamespace(rmsnorm=lambda x, w, eps: x),
raising=False,
)
# Case 1: CUDA not available.
monkeypatch.setattr(torch.cuda, "is_available", lambda: False)
assert _oink_ops.is_oink_available_for_device(0) is False
# Case 2: CUDA available but < SM100.
monkeypatch.setattr(torch.cuda, "is_available", lambda: True)
monkeypatch.setattr(torch.cuda, "get_device_capability", lambda idx: (9, 0))
assert _oink_ops.is_oink_available_for_device(0) is False
# Case 3: CUDA available and SM100, rmsnorm op registered.
monkeypatch.setattr(torch.cuda, "get_device_capability", lambda idx: (10, 0))
assert _oink_ops.is_oink_available_for_device(0) is True
# fused op presence probe
assert _oink_ops.has_fused_add_rms_norm() is False
monkeypatch.setattr(
torch.ops,
"oink",
types.SimpleNamespace(
rmsnorm=lambda x, w, eps: x,
fused_add_rms_norm=lambda x, residual, w, eps: None,
# Use spawn to run function in fresh process with clean imports
# TODO migrate to spawn utility:
# https://github.com/vllm-project/vllm/issues/41415
ctx = multiprocessing.get_context("spawn")
process = ctx.Process(
target=_test_oink_availability_impl,
args=(
device_capability,
has_rmsnorm,
has_fused_add_rms_norm,
expected_available,
expected_fused,
),
raising=False,
)
assert _oink_ops.has_fused_add_rms_norm() is True
process.start()
process.join()
if process.exitcode != 0:
raise AssertionError(
f"Subprocess test failed with exit code {process.exitcode}"
)
def test_can_view_as_2d_stride_guard():
# Import the helper from the layernorm module.
from vllm.model_executor.layers.layernorm import _can_view_as_2d
# No global import
import torch
# Import the helper from the kernels module.
from vllm.kernels.oink_ops import _can_view_as_2d
x = torch.zeros((2, 3, 4))
assert _can_view_as_2d(x) is True
+37 -3
View File
@@ -1295,11 +1295,14 @@ def test_ir_op_priority_default():
# Assert default is applied to ops
priority_config = IrOpPriorityConfig.with_default(["vllm_c", "native"])
assert priority_config.rms_norm == ["vllm_c", "native"]
assert priority_config.fused_add_rms_norm == ["vllm_c", "native"]
# Assert single ops override the default
assert IrOpPriorityConfig.with_default(
["vllm_c", "native"], rms_norm=["oink", "native"]
) == IrOpPriorityConfig(rms_norm=["oink", "native"])
priority_config = IrOpPriorityConfig.with_default(
["native"], rms_norm=["oink", "native"]
)
assert priority_config.rms_norm == ["oink", "native"]
assert priority_config.fused_add_rms_norm == ["native"]
def test_ir_op_priority_str():
@@ -1318,3 +1321,34 @@ def test_ir_op_priority_str():
with pytest.raises(pydantic.ValidationError):
# must be list of only strings
priority_config = IrOpPriorityConfig(rms_norm=["vllm_c", 4, "native"])
def test_ir_op_priority_ctx():
"""Test that the priority-setting context sets priority correctly."""
from vllm import ir
from vllm.config.kernel import IrOpPriorityConfig
priority = IrOpPriorityConfig.with_default(["native"], rms_norm=["vllm_c"])
priority2 = IrOpPriorityConfig.with_default(
["native"], fused_add_rms_norm=["vllm_c"]
)
with priority.set_priority():
assert ir.ops.rms_norm.get_priority() == ["vllm_c", "native"]
assert ir.ops.fused_add_rms_norm.get_priority() == ["native"]
with priority2.set_priority():
assert ir.ops.rms_norm.get_priority() == ["native"]
assert ir.ops.fused_add_rms_norm.get_priority() == ["vllm_c", "native"]
# context restored
assert ir.ops.rms_norm.get_priority() == ["vllm_c", "native"]
assert ir.ops.fused_add_rms_norm.get_priority() == ["native"]
with pytest.raises(ValueError), priority2.set_priority():
assert ir.ops.rms_norm.get_priority() == ["native"]
assert ir.ops.fused_add_rms_norm.get_priority() == ["vllm_c", "native"]
raise ValueError
# context restored even after exception
assert ir.ops.rms_norm.get_priority() == ["vllm_c", "native"]
assert ir.ops.fused_add_rms_norm.get_priority() == ["native"]
@@ -12,7 +12,7 @@ import torch
from utils import skip_unsupported
from vllm.model_executor.layers.batch_invariant import rms_norm as triton_rms_norm
from vllm.model_executor.layers.layernorm import RMSNorm, fused_add_rms_norm
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.platforms import current_platform
DEVICE_TYPE = current_platform.device_type
@@ -105,6 +105,12 @@ def test_fused_add_rms_norm_batch_invariant_residual_path(
dim=0,
)
def fused_add_rms_norm(x, residual, w, e) -> tuple[torch.Tensor, torch.Tensor]:
import vllm._custom_ops as ops
ops.fused_add_rms_norm(x, residual, w, e)
return x, residual
out_single, residual_out_single = fused_add_rms_norm(
x_single.clone(),
residual_single.clone(),