[vLLM IR] 2/N fused_add_rms_norm and maybe_inplace overload (#36823)
Signed-off-by: Luka Govedič <lgovedic@redhat.com> Signed-off-by: Luka Govedič <ProExpertProg@users.noreply.github.com>
This commit is contained in:
@@ -12,10 +12,17 @@ from torch._ops import OpOverload, OpOverloadPacket
|
||||
from torch.fx._utils import lazy_format_graph_code
|
||||
|
||||
from vllm.compilation.passes.fx_utils import find_op_nodes
|
||||
from vllm.compilation.passes.inductor_pass import InductorPass
|
||||
from vllm.compilation.passes.inductor_pass import (
|
||||
InductorPass,
|
||||
pass_context,
|
||||
)
|
||||
from vllm.compilation.passes.ir.inplace_functionalization import (
|
||||
VllmIRInplaceFunctionalizationPass,
|
||||
)
|
||||
from vllm.compilation.passes.pass_manager import with_pattern_match_debug
|
||||
from vllm.compilation.passes.vllm_inductor_pass import VllmInductorPass
|
||||
from vllm.config import VllmConfig, get_current_vllm_config
|
||||
from vllm.config.utils import Range
|
||||
from vllm.logger import init_logger
|
||||
|
||||
logger = init_logger("vllm.tests.compile.backend")
|
||||
@@ -53,11 +60,17 @@ class TestBackend:
|
||||
self.custom_passes = list(passes)
|
||||
vllm_config = get_current_vllm_config()
|
||||
compile_config = vllm_config.compilation_config
|
||||
self.range = Range(1, vllm_config.scheduler_config.max_num_batched_tokens)
|
||||
# Deepcopy to allow multiple TestBackend instances to use the same VllmConfig
|
||||
self.inductor_config = deepcopy(compile_config.inductor_compile_config)
|
||||
self.inductor_config["force_disable_caches"] = True
|
||||
self.inductor_config["post_grad_custom_post_pass"] = self.post_pass
|
||||
|
||||
# Add VllmIRInplaceFunctionalizationPass as pre-grad pass by default
|
||||
self.inductor_config["pre_grad_custom_pass"] = (
|
||||
VllmIRInplaceFunctionalizationPass(vllm_config)
|
||||
)
|
||||
|
||||
if debug_dump_path := vllm_config.compile_debug_dump_path():
|
||||
logger.debug("Dumping depyf output to %s", debug_dump_path)
|
||||
self.debug_ctx = depyf.prepare_debug(debug_dump_path.as_posix())
|
||||
@@ -68,7 +81,7 @@ class TestBackend:
|
||||
self.graph_pre_compile = deepcopy(graph)
|
||||
from torch._inductor.compile_fx import compile_fx
|
||||
|
||||
with self.debug_ctx:
|
||||
with self.debug_ctx, pass_context(self.range):
|
||||
return compile_fx(
|
||||
graph, example_inputs, config_patches=self.inductor_config
|
||||
)
|
||||
|
||||
@@ -88,14 +88,10 @@ class TestAllReduceRMSNormModel(torch.nn.Module):
|
||||
]
|
||||
|
||||
def ops_in_model(self):
|
||||
return (
|
||||
[torch.ops.vllm_ir.rms_norm]
|
||||
+ [
|
||||
torch.ops._C.fused_add_rms_norm.default,
|
||||
]
|
||||
if RMSNorm.enabled()
|
||||
else []
|
||||
)
|
||||
return [
|
||||
torch.ops.vllm_ir.rms_norm,
|
||||
torch.ops.vllm_ir.fused_add_rms_norm,
|
||||
]
|
||||
|
||||
|
||||
class TestAllReduceRMSNormStaticQuantFP8Model(torch.nn.Module):
|
||||
@@ -152,16 +148,17 @@ class TestAllReduceRMSNormStaticQuantFP8Model(torch.nn.Module):
|
||||
def ops_in_model(self):
|
||||
if self.vllm_config.compilation_config.pass_config.fuse_norm_quant:
|
||||
return [torch.ops._C.fused_add_rms_norm_static_fp8_quant.default]
|
||||
elif RMSNorm.enabled():
|
||||
return [
|
||||
torch.ops._C.fused_add_rms_norm.default,
|
||||
]
|
||||
elif any(layer.is_quant_fp8_enabled() for layer in self.fp8_linear_layers):
|
||||
return [
|
||||
torch.ops._C.static_scaled_fp8_quant.default,
|
||||
]
|
||||
else:
|
||||
return []
|
||||
quant_ops = (
|
||||
[torch.ops._C.static_scaled_fp8_quant.default]
|
||||
if any(layer.is_quant_fp8_enabled() for layer in self.fp8_linear_layers)
|
||||
else [torch.ops.aten.reciprocal]
|
||||
)
|
||||
return [
|
||||
torch.ops.vllm_ir.rms_norm,
|
||||
torch.ops.vllm_ir.fused_add_rms_norm,
|
||||
*quant_ops,
|
||||
]
|
||||
|
||||
|
||||
@multi_gpu_test(num_gpus=2)
|
||||
|
||||
@@ -0,0 +1,412 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Comprehensive tests for UnsafeCloneEliminationPass.
|
||||
|
||||
This test suite exercises all possible valid FX graph patterns involving clones:
|
||||
1. Clone with no users (dead code)
|
||||
2. Clone with read-only users
|
||||
3. Clone with mutation users
|
||||
4. Clone of graph input
|
||||
5. Clone with original used after mutation
|
||||
6. Clone chains
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from torch import fx
|
||||
from torch.fx.experimental.proxy_tensor import make_fx
|
||||
|
||||
from vllm.compilation.passes.fx_utils import find_op_nodes
|
||||
from vllm.compilation.passes.inductor_pass import get_pass_context, pass_context
|
||||
from vllm.compilation.passes.ir.clone_elimination import (
|
||||
UnsafeCloneEliminationPass,
|
||||
user_writes_to_node,
|
||||
)
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.config.utils import Range
|
||||
|
||||
|
||||
def count_clones(graph: fx.Graph) -> int:
|
||||
"""Count clone nodes in a graph."""
|
||||
return len(list(find_op_nodes(torch.ops.aten.clone.default, graph)))
|
||||
|
||||
|
||||
@pytest.fixture(scope="function")
|
||||
def clone_cleanup_pass():
|
||||
return UnsafeCloneEliminationPass(VllmConfig())
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def setup_pass_context():
|
||||
"""Set up pass context for each test."""
|
||||
with pass_context(compile_range=Range(1, 8192)):
|
||||
yield
|
||||
|
||||
|
||||
class TestCloneCleanup:
|
||||
"""Test UnsafeCloneEliminationPass behavior on various graph patterns."""
|
||||
|
||||
def test_remove_clone_readonly_users(self, clone_cleanup_pass):
|
||||
"""Clone with only read-only users should be removed."""
|
||||
|
||||
def f(x: torch.Tensor) -> torch.Tensor:
|
||||
x_clone = x.clone()
|
||||
return x_clone + 1
|
||||
|
||||
inp = torch.randn(2, 3)
|
||||
graph_module = make_fx(f)(inp)
|
||||
assert count_clones(graph_module.graph) == 1
|
||||
|
||||
expected = graph_module(inp)
|
||||
clone_cleanup_pass(graph_module.graph)
|
||||
graph_module.recompile()
|
||||
actual = graph_module(inp)
|
||||
|
||||
assert count_clones(graph_module.graph) == 0
|
||||
torch.testing.assert_close(actual, expected)
|
||||
|
||||
def test_keep_clone_with_mutation_and_original_used_after(self, clone_cleanup_pass):
|
||||
"""Clone must be kept if it's mutated AND original is used after mutation."""
|
||||
|
||||
def f(x: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
x = x.relu() # not a graph param
|
||||
x_clone = x.clone()
|
||||
x_clone.add_(1)
|
||||
return x, x_clone
|
||||
|
||||
inp = torch.randn(2, 3)
|
||||
graph_module = make_fx(f)(inp)
|
||||
assert count_clones(graph_module.graph) == 1
|
||||
|
||||
expected = graph_module(inp)
|
||||
clone_cleanup_pass(graph_module.graph)
|
||||
graph_module.recompile()
|
||||
actual = graph_module(inp)
|
||||
|
||||
# Clone should be KEPT because original is used after mutation
|
||||
assert count_clones(graph_module.graph) == 1
|
||||
torch.testing.assert_close(actual[0], expected[0])
|
||||
torch.testing.assert_close(actual[1], expected[1])
|
||||
|
||||
def test_remove_clone_with_mutation_no_original_use(self, clone_cleanup_pass):
|
||||
"""Clone can be removed if it's mutated but original is not used after."""
|
||||
|
||||
def f(x: torch.Tensor) -> torch.Tensor:
|
||||
x = x.relu() # not a graph param
|
||||
x_clone = x.clone()
|
||||
x_clone.add_(1)
|
||||
return x_clone
|
||||
|
||||
inp = torch.randn(2, 3)
|
||||
graph_module = make_fx(f)(inp)
|
||||
assert count_clones(graph_module.graph) == 1
|
||||
|
||||
expected = graph_module(inp)
|
||||
clone_cleanup_pass(graph_module.graph)
|
||||
graph_module.recompile()
|
||||
actual = graph_module(inp)
|
||||
|
||||
assert count_clones(graph_module.graph) == 0
|
||||
torch.testing.assert_close(actual, expected)
|
||||
|
||||
def test_clone_chain(self, clone_cleanup_pass):
|
||||
"""Test handling of clone chains: x -> clone1 -> clone2."""
|
||||
|
||||
def f(x: torch.Tensor) -> torch.Tensor:
|
||||
x = x.relu() # not a graph param
|
||||
x1 = x.clone()
|
||||
x2 = x1.clone()
|
||||
return x2 + 1
|
||||
|
||||
inp = torch.randn(2, 3)
|
||||
graph_module = make_fx(f)(inp)
|
||||
assert count_clones(graph_module.graph) == 2
|
||||
|
||||
expected = graph_module(inp)
|
||||
clone_cleanup_pass(graph_module.graph)
|
||||
graph_module.recompile()
|
||||
actual = graph_module(inp)
|
||||
|
||||
# Both clones should be removed
|
||||
assert count_clones(graph_module.graph) == 0
|
||||
torch.testing.assert_close(actual, expected)
|
||||
|
||||
def test_multiple_clones_of_same_input(self, clone_cleanup_pass):
|
||||
"""Test multiple independent clones of the same input."""
|
||||
|
||||
def f(x: torch.Tensor) -> torch.Tensor:
|
||||
x1 = x.clone()
|
||||
x2 = x.clone()
|
||||
return x1 + x2
|
||||
|
||||
inp = torch.randn(2, 3)
|
||||
graph_module = make_fx(f)(inp)
|
||||
assert count_clones(graph_module.graph) == 2
|
||||
|
||||
expected = graph_module(inp)
|
||||
clone_cleanup_pass(graph_module.graph)
|
||||
graph_module.recompile()
|
||||
actual = graph_module(inp)
|
||||
|
||||
# Both clones should be removed (only readonly uses)
|
||||
assert count_clones(graph_module.graph) == 0
|
||||
torch.testing.assert_close(actual, expected)
|
||||
|
||||
def test_no_clones_in_graph(self, clone_cleanup_pass):
|
||||
"""Test pass behavior when graph has no clones."""
|
||||
|
||||
def f(x: torch.Tensor) -> torch.Tensor:
|
||||
return x + 1
|
||||
|
||||
inp = torch.randn(2, 3)
|
||||
graph_module = make_fx(f)(inp)
|
||||
assert count_clones(graph_module.graph) == 0
|
||||
|
||||
expected = graph_module(inp)
|
||||
clone_cleanup_pass(graph_module.graph)
|
||||
graph_module.recompile()
|
||||
actual = graph_module(inp)
|
||||
|
||||
assert count_clones(graph_module.graph) == 0
|
||||
torch.testing.assert_close(actual, expected)
|
||||
|
||||
def test_multiple_passes(self, clone_cleanup_pass):
|
||||
"""Test running the pass multiple times (should be idempotent)."""
|
||||
|
||||
def f(x: torch.Tensor) -> torch.Tensor:
|
||||
x1 = x.clone()
|
||||
return x1 + 1
|
||||
|
||||
inp = torch.randn(2, 3)
|
||||
graph_module = make_fx(f)(inp)
|
||||
assert count_clones(graph_module.graph) == 1
|
||||
|
||||
expected = graph_module(inp)
|
||||
|
||||
clone_cleanup_pass(graph_module.graph)
|
||||
assert count_clones(graph_module.graph) == 0
|
||||
graph_module.recompile()
|
||||
actual = graph_module(inp)
|
||||
torch.testing.assert_close(actual, expected)
|
||||
|
||||
clone_cleanup_pass(graph_module.graph)
|
||||
assert count_clones(graph_module.graph) == 0
|
||||
graph_module.recompile()
|
||||
actual = graph_module(inp)
|
||||
torch.testing.assert_close(actual, expected)
|
||||
|
||||
def test_output_node_no_write(self):
|
||||
"""Output nodes never write to their inputs."""
|
||||
|
||||
def f(x: torch.Tensor) -> torch.Tensor:
|
||||
return x
|
||||
|
||||
graph_module = make_fx(f)(torch.randn(2, 3))
|
||||
x_node = [n for n in graph_module.graph.nodes if n.op == "placeholder"][0]
|
||||
output_node = [n for n in graph_module.graph.nodes if n.op == "output"][0]
|
||||
|
||||
assert not user_writes_to_node(output_node, x_node)
|
||||
|
||||
def test_readonly_op_no_write(self):
|
||||
"""Readonly operations don't write to inputs."""
|
||||
|
||||
def f(x: torch.Tensor, y: torch.Tensor) -> torch.Tensor:
|
||||
return x + y
|
||||
|
||||
graph_module = make_fx(f)(torch.randn(2, 3), torch.randn(2, 3))
|
||||
placeholders = [n for n in graph_module.graph.nodes if n.op == "placeholder"]
|
||||
add_node = [
|
||||
n
|
||||
for n in graph_module.graph.nodes
|
||||
if n.op == "call_function" and n.target == torch.ops.aten.add.Tensor
|
||||
][0]
|
||||
|
||||
assert not user_writes_to_node(add_node, placeholders[0])
|
||||
assert not user_writes_to_node(add_node, placeholders[1])
|
||||
|
||||
def test_inplace_op_writes(self):
|
||||
"""Inplace operations write to first argument."""
|
||||
|
||||
def f(x: torch.Tensor, y: torch.Tensor) -> torch.Tensor:
|
||||
x.add_(y)
|
||||
return x
|
||||
|
||||
graph_module = make_fx(f)(torch.randn(2, 3), torch.randn(2, 3))
|
||||
placeholders = [n for n in graph_module.graph.nodes if n.op == "placeholder"]
|
||||
add_node = [
|
||||
n
|
||||
for n in graph_module.graph.nodes
|
||||
if n.op == "call_function" and "add_" in str(n.target)
|
||||
][0]
|
||||
|
||||
# add_ writes to first arg but not second
|
||||
assert user_writes_to_node(add_node, placeholders[0])
|
||||
assert not user_writes_to_node(add_node, placeholders[1])
|
||||
|
||||
def test_copy_writes(self):
|
||||
"""copy_ operation writes to first argument."""
|
||||
|
||||
def f(x: torch.Tensor, y: torch.Tensor) -> torch.Tensor:
|
||||
x.copy_(y)
|
||||
return x
|
||||
|
||||
graph_module = make_fx(f)(torch.randn(2, 3), torch.randn(2, 3))
|
||||
placeholders = [n for n in graph_module.graph.nodes if n.op == "placeholder"]
|
||||
copy_node = [
|
||||
n
|
||||
for n in graph_module.graph.nodes
|
||||
if n.op == "call_function" and "copy_" in str(n.target)
|
||||
][0]
|
||||
|
||||
assert user_writes_to_node(copy_node, placeholders[0])
|
||||
assert not user_writes_to_node(copy_node, placeholders[1])
|
||||
|
||||
def test_auto_functionalized_not_a_write(self):
|
||||
"""auto_functionalized ops are follow-up uses, not writes."""
|
||||
from torch._higher_order_ops.auto_functionalize import auto_functionalized
|
||||
|
||||
def f(x: torch.Tensor) -> torch.Tensor:
|
||||
return x
|
||||
|
||||
graph_module = make_fx(f)(torch.randn(2, 3))
|
||||
x_node = [n for n in graph_module.graph.nodes if n.op == "placeholder"][0]
|
||||
|
||||
# Create an auto_functionalized node in the graph
|
||||
with graph_module.graph.inserting_before(None):
|
||||
af_node = graph_module.graph.call_function(
|
||||
auto_functionalized, kwargs={"input": x_node}
|
||||
)
|
||||
|
||||
# auto_functionalized should not be treated as a write
|
||||
assert not user_writes_to_node(af_node, x_node)
|
||||
|
||||
def test_higher_order_op_conservatively_writes(self):
|
||||
"""Other higher-order operators are conservatively treated as writes."""
|
||||
from torch._ops import HigherOrderOperator
|
||||
|
||||
def f(x: torch.Tensor) -> torch.Tensor:
|
||||
return x
|
||||
|
||||
graph_module = make_fx(f)(torch.randn(2, 3))
|
||||
x_node = [n for n in graph_module.graph.nodes if n.op == "placeholder"][0]
|
||||
|
||||
# Create a concrete higher-order operator subclass
|
||||
class MockHigherOrderOp(HigherOrderOperator):
|
||||
def __call__(self, *args, **kwargs):
|
||||
return args[0] if args else None
|
||||
|
||||
mock_hoo = MockHigherOrderOp("mock_higher_order_op")
|
||||
|
||||
with graph_module.graph.inserting_before(None):
|
||||
hoo_node = graph_module.graph.call_function(mock_hoo, args=(x_node,))
|
||||
|
||||
# Should be conservative and assume it could write
|
||||
assert user_writes_to_node(hoo_node, x_node)
|
||||
|
||||
|
||||
class TestCloneCleanupWithDonatedInputs:
|
||||
"""Test UnsafeCloneEliminationPass with donated input tracking via PassContext."""
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def setup_pass_context(self):
|
||||
"""Set up pass context for each test."""
|
||||
with pass_context(compile_range=Range(1, 8192)):
|
||||
yield
|
||||
|
||||
def test_donated_input_clone_removed(self, clone_cleanup_pass):
|
||||
"""Clone of donated input should be removed."""
|
||||
|
||||
def f(x: torch.Tensor) -> torch.Tensor:
|
||||
x_clone = x.clone()
|
||||
x_clone.add_(1)
|
||||
return x_clone
|
||||
|
||||
inp = torch.randn(2, 3)
|
||||
graph_module = make_fx(f)(inp)
|
||||
assert count_clones(graph_module.graph) == 1
|
||||
|
||||
# Mark first parameter as donated
|
||||
get_pass_context().donated_input_ids = {0}
|
||||
|
||||
expected = graph_module(inp.clone())
|
||||
clone_cleanup_pass(graph_module.graph)
|
||||
graph_module.recompile()
|
||||
|
||||
# Clone should be removed since input is donated
|
||||
assert count_clones(graph_module.graph) == 0
|
||||
|
||||
# Input can be mutated (donated)
|
||||
inp_copy = inp.clone()
|
||||
actual = graph_module(inp_copy)
|
||||
torch.testing.assert_close(actual, expected)
|
||||
|
||||
def test_non_donated_input_clone_kept(self, clone_cleanup_pass):
|
||||
"""Clone of non-donated input with mutation should be kept."""
|
||||
|
||||
def f(x: torch.Tensor, y: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
x_clone = x.clone()
|
||||
x_clone.add_(1)
|
||||
return x, x_clone
|
||||
|
||||
inp_x = torch.randn(2, 3)
|
||||
inp_y = torch.randn(2, 3)
|
||||
graph_module = make_fx(f)(inp_x, inp_y)
|
||||
assert count_clones(graph_module.graph) == 1
|
||||
|
||||
# No donated inputs
|
||||
get_pass_context().donated_input_ids = set()
|
||||
|
||||
expected = graph_module(inp_x.clone(), inp_y.clone())
|
||||
clone_cleanup_pass(graph_module.graph)
|
||||
graph_module.recompile()
|
||||
|
||||
# Clone should be kept since input is not donated and original is used
|
||||
assert count_clones(graph_module.graph) == 1
|
||||
|
||||
# Verify inputs are not mutated
|
||||
inp_x_before = inp_x.clone()
|
||||
inp_y_before = inp_y.clone()
|
||||
actual = graph_module(inp_x, inp_y)
|
||||
torch.testing.assert_close(
|
||||
inp_x, inp_x_before, msg="Input x should not be mutated"
|
||||
)
|
||||
torch.testing.assert_close(
|
||||
inp_y, inp_y_before, msg="Input y should not be mutated"
|
||||
)
|
||||
torch.testing.assert_close(actual[0], expected[0])
|
||||
torch.testing.assert_close(actual[1], expected[1])
|
||||
|
||||
def test_mixed_donated_inputs(self, clone_cleanup_pass):
|
||||
"""Test with some inputs donated and some not."""
|
||||
|
||||
def f(x: torch.Tensor, y: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
x_clone = x.clone()
|
||||
x_clone.add_(1)
|
||||
y_clone = y.clone()
|
||||
y_clone.add_(2)
|
||||
return x_clone, y_clone
|
||||
|
||||
inp_x = torch.randn(2, 3)
|
||||
inp_y = torch.randn(2, 3)
|
||||
graph_module = make_fx(f)(inp_x, inp_y)
|
||||
assert count_clones(graph_module.graph) == 2
|
||||
|
||||
# Only x is donated
|
||||
get_pass_context().donated_input_ids = {0}
|
||||
|
||||
expected = graph_module(inp_x.clone(), inp_y.clone())
|
||||
clone_cleanup_pass(graph_module.graph)
|
||||
graph_module.recompile()
|
||||
|
||||
# x_clone removed (x is donated), y_clone kept (y is not donated)
|
||||
assert count_clones(graph_module.graph) == 1
|
||||
|
||||
# Verify y is not mutated (x can be mutated since it's donated)
|
||||
inp_y_before = inp_y.clone()
|
||||
actual = graph_module(inp_x.clone(), inp_y)
|
||||
torch.testing.assert_close(
|
||||
inp_y, inp_y_before, msg="Input y should not be mutated"
|
||||
)
|
||||
torch.testing.assert_close(actual[0], expected[0])
|
||||
torch.testing.assert_close(actual[1], expected[1])
|
||||
@@ -0,0 +1,465 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Tests for IR inplace functionalization pass integration.
|
||||
|
||||
This test suite verifies that the inplace functionalization pass, lowering pass,
|
||||
and clone cleanup pass work together correctly with donated buffer tracking.
|
||||
"""
|
||||
|
||||
from collections.abc import Callable
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
import torch._dynamo.exc
|
||||
from torch import nn
|
||||
|
||||
import vllm.kernels # noqa: F401 to register kernels
|
||||
from vllm.compilation.passes.inductor_pass import InductorPass, get_pass_context
|
||||
from vllm.compilation.passes.ir.clone_elimination import (
|
||||
UnsafeCloneEliminationPass,
|
||||
)
|
||||
from vllm.compilation.passes.ir.inplace_functionalization import (
|
||||
VllmIRInplaceFunctionalizationPass,
|
||||
)
|
||||
from vllm.compilation.passes.ir.lowering_pass import VllmIRLoweringPass
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.ir import ops
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import HAS_TRITON, tl, triton
|
||||
|
||||
from ...backend import TestBackend
|
||||
|
||||
|
||||
class StoreDonationInfoPass(InductorPass):
|
||||
def __init__(self):
|
||||
self.donated_input_ids_sets: list[set[int]] = []
|
||||
|
||||
def __call__(self, *args, **kwargs):
|
||||
ctx = get_pass_context()
|
||||
self.donated_input_ids_sets += [ctx.donated_input_ids]
|
||||
|
||||
|
||||
class MaybeInplaceModel(nn.Module):
|
||||
"""Model using only maybe_inplace variants."""
|
||||
|
||||
def __init__(self, hidden_size=16):
|
||||
super().__init__()
|
||||
self.weight1 = nn.Parameter(torch.ones(hidden_size, dtype=torch.bfloat16))
|
||||
self.weight2 = nn.Parameter(torch.ones(hidden_size, dtype=torch.bfloat16))
|
||||
|
||||
def forward(
|
||||
self, x: torch.Tensor, residual1: torch.Tensor, residual2: torch.Tensor
|
||||
):
|
||||
# First maybe_inplace - x & residual1 are donated
|
||||
x_normed1, residual_out1 = ops.fused_add_rms_norm.maybe_inplace(
|
||||
x, residual1, self.weight1, 1e-5
|
||||
)
|
||||
# Second maybe_inplace - residual2 is donated
|
||||
x_normed2, residual_out2 = ops.fused_add_rms_norm.maybe_inplace(
|
||||
x_normed1, residual2, self.weight2, 1e-5
|
||||
)
|
||||
return x_normed2, residual_out1, residual_out2
|
||||
|
||||
|
||||
class FunctionalModel(nn.Module):
|
||||
"""Model using only functional (default) variants."""
|
||||
|
||||
def __init__(self, hidden_size=16):
|
||||
super().__init__()
|
||||
self.weight1 = nn.Parameter(torch.ones(hidden_size, dtype=torch.bfloat16))
|
||||
self.weight2 = nn.Parameter(torch.ones(hidden_size, dtype=torch.bfloat16))
|
||||
|
||||
def forward(
|
||||
self, x: torch.Tensor, residual1: torch.Tensor, residual2: torch.Tensor
|
||||
):
|
||||
# First functional - no donation
|
||||
x_normed1, residual_out1 = ops.fused_add_rms_norm(
|
||||
x, residual1, self.weight1, 1e-5
|
||||
)
|
||||
# Second functional - no donation
|
||||
x_normed2, residual_out2 = ops.fused_add_rms_norm(
|
||||
x_normed1, residual2, self.weight2, 1e-5
|
||||
)
|
||||
return x_normed2, residual_out1, residual_out2
|
||||
|
||||
|
||||
class MixedModel(nn.Module):
|
||||
"""Model mixing maybe_inplace and functional variants."""
|
||||
|
||||
def __init__(self, hidden_size=16):
|
||||
super().__init__()
|
||||
self.weight1 = nn.Parameter(torch.ones(hidden_size, dtype=torch.bfloat16))
|
||||
self.weight2 = nn.Parameter(torch.ones(hidden_size, dtype=torch.bfloat16))
|
||||
|
||||
def forward(
|
||||
self, x: torch.Tensor, residual1: torch.Tensor, residual2: torch.Tensor
|
||||
):
|
||||
# First maybe_inplace - x & residual1 are donated
|
||||
x_normed1, residual_out1 = ops.fused_add_rms_norm.maybe_inplace(
|
||||
x, residual1, self.weight1, 1e-5
|
||||
)
|
||||
# Second functional - no donation, x_normed1 must be preserved as it's returned
|
||||
x_normed2, residual_out2 = ops.fused_add_rms_norm(
|
||||
x_normed1, residual2, self.weight2, 1e-5
|
||||
)
|
||||
# Return both to prevent x_normed1 from being optimized away
|
||||
return x_normed1, x_normed2, residual_out1, residual_out2
|
||||
|
||||
|
||||
class ModelWithTritonAfterMaybeInplace(nn.Module):
|
||||
"""
|
||||
Model using maybe_inplace followed by a Triton kernel.
|
||||
Test clone elimination can handle Triton in the graph
|
||||
"""
|
||||
|
||||
def __init__(self, hidden_size=16):
|
||||
super().__init__()
|
||||
self.weight = nn.Parameter(torch.ones(hidden_size, dtype=torch.bfloat16))
|
||||
|
||||
@triton.jit
|
||||
def _triton_add_kernel(
|
||||
x_ptr,
|
||||
y_ptr,
|
||||
n_elements,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
):
|
||||
pid = tl.program_id(axis=0)
|
||||
block_start = pid * BLOCK_SIZE
|
||||
offsets = block_start + tl.arange(0, BLOCK_SIZE)
|
||||
mask = offsets < n_elements
|
||||
x = tl.load(x_ptr + offsets, mask=mask)
|
||||
y = x + 0.1
|
||||
tl.store(y_ptr + offsets, y, mask=mask)
|
||||
|
||||
def triton_add(x: torch.Tensor) -> torch.Tensor:
|
||||
"""Simple Triton add kernel."""
|
||||
y = torch.empty_like(x)
|
||||
n_elements = x.numel()
|
||||
grid = (triton.cdiv(n_elements, 256),)
|
||||
_triton_add_kernel[grid](x, y, n_elements, BLOCK_SIZE=256)
|
||||
return y
|
||||
|
||||
self.triton_add = triton_add
|
||||
|
||||
def forward(self, x: torch.Tensor, residual: torch.Tensor, residual2: torch.Tensor):
|
||||
x_normed, residual_out = ops.fused_add_rms_norm.maybe_inplace(
|
||||
x, residual, self.weight, 1e-5
|
||||
)
|
||||
|
||||
x_processed = self.triton_add(x_normed)
|
||||
|
||||
# x_processed does not need to be cloned, residual2 does
|
||||
x_normed2, residual_out2 = ops.fused_add_rms_norm(
|
||||
x_processed, residual2, self.weight, 1e-5
|
||||
)
|
||||
return x_normed2, residual_out2
|
||||
|
||||
|
||||
skipif_no_triton = pytest.mark.skipif(not HAS_TRITON, reason="Requires Triton")
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda_alike(),
|
||||
reason="Only test on cuda and rocm platform",
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"model_class,expected_functionalized,expected_donated,expected_clones",
|
||||
[
|
||||
# 2 inplace calls, all activations donated, all clones eliminated
|
||||
(MaybeInplaceModel, 2, 3, 0),
|
||||
# No inplace calls, no donations, 3 clones (one eliminated)
|
||||
(FunctionalModel, 0, 0, 3),
|
||||
# One inplace call, two donated activations, 2 clones
|
||||
(MixedModel, 1, 2, 2),
|
||||
# One inplace call, two donated, 1 clone remaining
|
||||
pytest.param(ModelWithTritonAfterMaybeInplace, 1, 2, 1, marks=skipif_no_triton),
|
||||
],
|
||||
)
|
||||
def test_inplace_functionalization(
|
||||
default_vllm_config: VllmConfig,
|
||||
model_class,
|
||||
expected_functionalized: int,
|
||||
expected_clones: int,
|
||||
expected_donated: int,
|
||||
):
|
||||
"""Test inplace functionalization, lowering, and clone cleanup."""
|
||||
torch.set_default_device(current_platform.device_type)
|
||||
|
||||
# Use vllm_c so inplace path is triggered
|
||||
default_vllm_config.kernel_config.ir_op_priority.fused_add_rms_norm = [
|
||||
"vllm_c",
|
||||
"native",
|
||||
]
|
||||
|
||||
# Create passes in order they run during compilation
|
||||
functionalization_pass = VllmIRInplaceFunctionalizationPass(default_vllm_config)
|
||||
lowering_pass = VllmIRLoweringPass(default_vllm_config)
|
||||
donated_info_pass = StoreDonationInfoPass()
|
||||
cleanup_pass = UnsafeCloneEliminationPass(default_vllm_config)
|
||||
|
||||
# Set up backend with pre-grad pass
|
||||
backend = TestBackend(lowering_pass, donated_info_pass, cleanup_pass)
|
||||
backend.inductor_config["pre_grad_custom_pass"] = functionalization_pass
|
||||
|
||||
model = model_class()
|
||||
x = torch.randn(8, 16, dtype=torch.bfloat16)
|
||||
residual1 = torch.randn(8, 16, dtype=torch.bfloat16)
|
||||
residual2 = torch.randn(8, 16, dtype=torch.bfloat16)
|
||||
|
||||
with default_vllm_config.kernel_config.ir_op_priority.set_priority():
|
||||
# Reference output without optimization
|
||||
ref_output = model(x.clone(), residual1.clone(), residual2.clone())
|
||||
|
||||
# Compile with inplace optimization
|
||||
compiled_model = torch.compile(model, backend=backend, fullgraph=True)
|
||||
output = compiled_model(x.clone(), residual1.clone(), residual2.clone())
|
||||
|
||||
# Verify correctness (relaxed tolerance for bfloat16)
|
||||
for i in range(len(ref_output)):
|
||||
torch.testing.assert_close(output[i], ref_output[i], rtol=1e-2, atol=1e-2)
|
||||
|
||||
# Verify expected number of ops were functionalized
|
||||
func_ops = functionalization_pass.functionalized_ops
|
||||
assert len(func_ops) == int(bool(expected_functionalized))
|
||||
if expected_functionalized > 0:
|
||||
assert "fused_add_rms_norm" in func_ops
|
||||
assert func_ops["fused_add_rms_norm"] == expected_functionalized
|
||||
|
||||
# Verify lowering happened (2 ops in all cases)
|
||||
assert "fused_add_rms_norm" in lowering_pass.selected_impls
|
||||
assert len(lowering_pass.selected_impls["fused_add_rms_norm"]) == 2
|
||||
assert all(
|
||||
provider == "vllm_c"
|
||||
for node, provider in lowering_pass.selected_impls["fused_add_rms_norm"].items()
|
||||
), lowering_pass.selected_impls
|
||||
|
||||
# Verify correct number of donated IDs
|
||||
assert len(donated_info_pass.donated_input_ids_sets) == 1
|
||||
assert len(donated_info_pass.donated_input_ids_sets[0]) == expected_donated
|
||||
|
||||
# Verify expected number of clones after cleanup
|
||||
actual_clones = backend.op_count(torch.ops.aten.clone.default, before=False)
|
||||
assert actual_clones == expected_clones, (
|
||||
f"Expected {expected_clones} clones, got {actual_clones}:"
|
||||
f"{backend.print_graphs()}"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda_alike(),
|
||||
reason="Only test on cuda and rocm platform",
|
||||
)
|
||||
def test_donated_buffer_context_propagation(default_vllm_config):
|
||||
"""Test that donated_input_ids propagates correctly through pass_context."""
|
||||
torch.set_default_device(current_platform.device_type)
|
||||
|
||||
# Create a custom backend that inspects pass_context in cleanup pass
|
||||
functionalization_pass = VllmIRInplaceFunctionalizationPass(default_vllm_config)
|
||||
lowering_pass = VllmIRLoweringPass(default_vllm_config)
|
||||
|
||||
donation_info_pass = StoreDonationInfoPass()
|
||||
cleanup_pass = UnsafeCloneEliminationPass(default_vllm_config)
|
||||
|
||||
backend = TestBackend(lowering_pass, donation_info_pass, cleanup_pass)
|
||||
backend.inductor_config["pre_grad_custom_pass"] = functionalization_pass
|
||||
|
||||
model = MaybeInplaceModel()
|
||||
x = torch.randn(8, 16, dtype=torch.bfloat16)
|
||||
residual1 = torch.randn(8, 16, dtype=torch.bfloat16)
|
||||
residual2 = torch.randn(8, 16, dtype=torch.bfloat16)
|
||||
|
||||
compiled_model = torch.compile(model, backend=backend, fullgraph=True)
|
||||
compiled_model(x.clone(), residual1.clone(), residual2.clone())
|
||||
|
||||
donated_ids_seen = donation_info_pass.donated_input_ids_sets
|
||||
# Verify donated_input_ids was set and propagated
|
||||
assert len(donated_ids_seen) == 1
|
||||
# Should have donated inputs (exact indices depend on AOTAutograd)
|
||||
assert len(donated_ids_seen[0]) == 3
|
||||
# All donated ids should be valid non-negative integers
|
||||
for idx in donated_ids_seen[0]:
|
||||
assert isinstance(idx, int) and idx >= 0, f"Invalid donated index: {idx}"
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda_alike(),
|
||||
reason="Only test on cuda and rocm platform",
|
||||
)
|
||||
def test_maybe_inplace_reuse_error(default_vllm_config):
|
||||
"""Test that reusing a donated activation input raises ValueError."""
|
||||
torch.set_default_device(current_platform.device_type)
|
||||
|
||||
class ReuseModel(nn.Module):
|
||||
"""Model that incorrectly reuses a donated activation input."""
|
||||
|
||||
def __init__(self, hidden_size=16):
|
||||
super().__init__()
|
||||
self.weight = nn.Parameter(torch.ones(hidden_size, dtype=torch.bfloat16))
|
||||
|
||||
def forward(self, x: torch.Tensor, residual: torch.Tensor):
|
||||
# x is donated to maybe_inplace
|
||||
x_normed, residual_out = ops.fused_add_rms_norm.maybe_inplace(
|
||||
x, residual, self.weight, 1e-5
|
||||
)
|
||||
# ERROR: x is used again after being donated
|
||||
return x_normed + x # This should raise ValueError
|
||||
|
||||
functionalization_pass = VllmIRInplaceFunctionalizationPass(default_vllm_config)
|
||||
lowering_pass = VllmIRLoweringPass(default_vllm_config)
|
||||
cleanup_pass = UnsafeCloneEliminationPass(default_vllm_config)
|
||||
|
||||
backend = TestBackend(lowering_pass, cleanup_pass)
|
||||
backend.inductor_config["pre_grad_custom_pass"] = functionalization_pass
|
||||
|
||||
model = ReuseModel()
|
||||
x = torch.randn(8, 16, dtype=torch.bfloat16)
|
||||
residual = torch.randn(8, 16, dtype=torch.bfloat16)
|
||||
|
||||
# Compilation should raise BackendCompilerFailed wrapping ValueError
|
||||
with pytest.raises(
|
||||
torch._dynamo.exc.BackendCompilerFailed,
|
||||
match="is used again after the node",
|
||||
):
|
||||
compiled_model = torch.compile(model, backend=backend, fullgraph=True)
|
||||
compiled_model(x.clone(), residual.clone())
|
||||
|
||||
|
||||
# Piecewise compilation tests with graph splitting
|
||||
|
||||
|
||||
@torch.library.custom_op("vllm::test_split_marker", mutates_args=())
|
||||
def test_split_marker(x: torch.Tensor) -> torch.Tensor:
|
||||
"""Identity op that marks a split point for piecewise compilation."""
|
||||
return x.clone()
|
||||
|
||||
|
||||
@test_split_marker.register_fake
|
||||
def _fake_split_marker(x: torch.Tensor) -> torch.Tensor:
|
||||
return torch.empty_like(x)
|
||||
|
||||
|
||||
class TransformerBlockWithSplits(nn.Module):
|
||||
"""Transformer block with explicit split points for piecewise compilation."""
|
||||
|
||||
def __init__(self, hidden_size=32, intermediate_size=128):
|
||||
super().__init__()
|
||||
self.hidden_size = hidden_size
|
||||
self.intermediate_size = intermediate_size
|
||||
|
||||
# Attention-like projection
|
||||
self.attn_proj = nn.Linear(
|
||||
hidden_size, hidden_size, bias=False, dtype=torch.bfloat16
|
||||
)
|
||||
|
||||
# Post-attention norm
|
||||
self.post_attn_norm = nn.Parameter(
|
||||
torch.ones(hidden_size, dtype=torch.bfloat16)
|
||||
)
|
||||
|
||||
# MLP
|
||||
self.gate_proj = nn.Linear(
|
||||
hidden_size, intermediate_size, bias=False, dtype=torch.bfloat16
|
||||
)
|
||||
self.up_proj = nn.Linear(
|
||||
hidden_size, intermediate_size, bias=False, dtype=torch.bfloat16
|
||||
)
|
||||
self.down_proj = nn.Linear(
|
||||
intermediate_size, hidden_size, bias=False, dtype=torch.bfloat16
|
||||
)
|
||||
|
||||
# Post-MLP norm
|
||||
self.post_mlp_norm = nn.Parameter(torch.ones(hidden_size, dtype=torch.bfloat16))
|
||||
|
||||
def forward(self, x: torch.Tensor):
|
||||
# Attention block with residual
|
||||
residual1 = x
|
||||
attn_out = self.attn_proj(x)
|
||||
|
||||
# Fused add + norm (maybe_inplace: residual1 is donated)
|
||||
normed1, residual1 = ops.fused_add_rms_norm.maybe_inplace(
|
||||
attn_out, residual1, self.post_attn_norm, 1e-5
|
||||
)
|
||||
|
||||
# Force a graph split here
|
||||
normed1 = torch.ops.vllm.test_split_marker(normed1)
|
||||
|
||||
# MLP block
|
||||
gate = self.gate_proj(normed1)
|
||||
up = self.up_proj(normed1)
|
||||
mlp_out = self.down_proj(gate * torch.nn.functional.silu(up))
|
||||
|
||||
# Fused add + norm (maybe_inplace: residual1 is donated)
|
||||
normed2, residual2 = ops.fused_add_rms_norm.maybe_inplace(
|
||||
mlp_out, residual1, self.post_mlp_norm, 1e-5
|
||||
)
|
||||
|
||||
return normed2, residual2
|
||||
|
||||
|
||||
def with_dyn_arg(fn: Callable, arg_index: int, dim_index: int):
|
||||
def inner(*args):
|
||||
torch._dynamo.mark_dynamic(args[arg_index], dim_index)
|
||||
return fn(*args)
|
||||
|
||||
return inner
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda_alike(),
|
||||
reason="Only test on cuda and rocm platform",
|
||||
)
|
||||
def test_piecewise_compilation_with_donated_buffers(monkeypatch, fresh_vllm_cache):
|
||||
"""
|
||||
Test piecewise compilation with donated buffers across graph splits.
|
||||
Utilizes a custom splitting op. Uses fresh cache to avoid compilation caching.
|
||||
"""
|
||||
torch.set_default_device(current_platform.device_type)
|
||||
|
||||
# Disable compilation cache to avoid serialization issues
|
||||
monkeypatch.setenv("VLLM_DISABLE_COMPILE_CACHE", "1")
|
||||
|
||||
from vllm.compilation.backends import VllmBackend
|
||||
from vllm.config import CompilationConfig, VllmConfig
|
||||
|
||||
# Create config with custom splitting op
|
||||
store_donation_info = StoreDonationInfoPass()
|
||||
vllm_config = VllmConfig(
|
||||
compilation_config=CompilationConfig(
|
||||
custom_ops=["all"],
|
||||
splitting_ops=["vllm::test_split_marker"],
|
||||
inductor_compile_config={"post_grad_custom_post_pass": store_donation_info},
|
||||
)
|
||||
)
|
||||
|
||||
backend = VllmBackend(vllm_config)
|
||||
|
||||
model = TransformerBlockWithSplits()
|
||||
x = torch.randn(8, 32, dtype=torch.bfloat16)
|
||||
|
||||
# Reference output
|
||||
ref_output = with_dyn_arg(model, 0, 0)(x.clone())
|
||||
|
||||
# Compile with piecewise compilation (graph will split at split_marker)
|
||||
compiled_model = torch.compile(model, backend=backend, fullgraph=False)
|
||||
output = with_dyn_arg(compiled_model, 0, 0)(x.clone())
|
||||
|
||||
# Verify correctness (relaxed tolerance for bfloat16)
|
||||
torch.testing.assert_close(output[0], ref_output[0], rtol=1e-2, atol=1e-2)
|
||||
torch.testing.assert_close(output[1], ref_output[1], rtol=1e-2, atol=1e-2)
|
||||
|
||||
# Verify the model was split into multiple submodules
|
||||
assert hasattr(backend, "split_gm"), "Backend should have split graph module"
|
||||
|
||||
# Should have at least 2 submodules (split by test_split_marker op)
|
||||
submodules = list(backend.split_gm.named_children())
|
||||
num_submodules = len(submodules)
|
||||
assert num_submodules >= 2, (
|
||||
f"Expected at least 2 submodules (split), got {num_submodules}"
|
||||
)
|
||||
|
||||
# Check that donation info was propagated correctly
|
||||
donated_inputs_sets = store_donation_info.donated_input_ids_sets
|
||||
assert len(donated_inputs_sets) == 2
|
||||
assert len(donated_inputs_sets[0]) == 1
|
||||
assert len(donated_inputs_sets[1]) == 1
|
||||
@@ -126,7 +126,7 @@ class TestFusedAddRMSNorm(torch.nn.Module):
|
||||
if TEST_FP8 and do_fusion:
|
||||
return [torch.ops._C.fused_add_rms_norm_static_fp8_quant.default]
|
||||
else:
|
||||
return [torch.ops._C.fused_add_rms_norm.default]
|
||||
return []
|
||||
|
||||
def ops_not_in_model(self):
|
||||
return []
|
||||
|
||||
@@ -59,7 +59,7 @@ class TestModel(torch.nn.Module):
|
||||
|
||||
def ops_in_model_before(self):
|
||||
return [
|
||||
rocm_aiter_ops.get_rmsnorm_fused_add_op(),
|
||||
torch.ops.vllm_ir.fused_add_rms_norm,
|
||||
torch.ops.aten.constant_pad_nd,
|
||||
]
|
||||
|
||||
|
||||
@@ -17,7 +17,6 @@ from vllm.compilation.passes.fusion.rms_quant_fusion import (
|
||||
FusedRMSQuantKey,
|
||||
RMSNormQuantFusionPass,
|
||||
)
|
||||
from vllm.compilation.passes.fx_utils import find_op_nodes
|
||||
from vllm.compilation.passes.utility.noop_elimination import NoOpEliminationPass
|
||||
from vllm.compilation.passes.utility.post_cleanup import PostCleanupPass
|
||||
from vllm.config import (
|
||||
@@ -243,9 +242,10 @@ class TestModel(torch.nn.Module):
|
||||
]
|
||||
|
||||
def ops_in_model_before_partial(self):
|
||||
return [torch.ops.vllm_ir.rms_norm] + (
|
||||
[RMS_ADD_OP] if self.enable_rms_norm_custom_op else [torch.ops.aten.rsqrt]
|
||||
)
|
||||
return [
|
||||
torch.ops.vllm_ir.rms_norm,
|
||||
torch.ops.vllm_ir.fused_add_rms_norm.default,
|
||||
]
|
||||
|
||||
|
||||
def _run_fusion_test(
|
||||
@@ -383,17 +383,6 @@ def test_fusion_rmsnorm_quant(
|
||||
model.ops_in_model_before_partial(), fully_replaced=False
|
||||
)
|
||||
|
||||
# If RMSNorm custom op is disabled (native/torch impl used),
|
||||
# there's a risk that the fused add doesn't get included in the
|
||||
# replacement and only the rms part gets fused with quant.
|
||||
# Hence, we check only 2 add nodes are left (final fused rmsnorm add).
|
||||
if not enable_rms_norm_custom_op:
|
||||
n_add_nodes = lambda g: sum(1 for _ in find_op_nodes(torch.ops.aten.add, g))
|
||||
# rms_norm is IR, not included
|
||||
# 6 = 3x2 (3xRMS_ADD, 2 each)
|
||||
assert n_add_nodes(backend.graph_pre_pass) == 6
|
||||
assert n_add_nodes(backend.graph_post_pass) == 2
|
||||
|
||||
|
||||
@pytest.mark.parametrize("dtype", [torch.bfloat16])
|
||||
@pytest.mark.parametrize("hidden_size", [256])
|
||||
|
||||
@@ -0,0 +1,91 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import torch
|
||||
from torch import Tensor
|
||||
from torch.fx.experimental.proxy_tensor import make_fx
|
||||
|
||||
import vllm.ir.op
|
||||
from vllm.ir.op import IrOp, IrOpInplaceOverload
|
||||
|
||||
|
||||
@vllm.ir.register_op(allow_inplace=True)
|
||||
def _custom_mm2(x: Tensor, w: Tensor) -> Tensor:
|
||||
return x @ w
|
||||
|
||||
|
||||
@_custom_mm2.register_impl("regular")
|
||||
def _custom_mm2_regular(x: Tensor, w: Tensor) -> Tensor:
|
||||
return x @ w + 1
|
||||
|
||||
|
||||
@_custom_mm2.register_impl("inplace", inplace=True)
|
||||
def _custom_mm2_inplace(x: Tensor, w: Tensor) -> Tensor:
|
||||
x.copy_(x @ w + 2)
|
||||
return x
|
||||
|
||||
|
||||
class TestInplaceOp:
|
||||
def test_registration(self):
|
||||
# Test that the inplace op is registered correctly.
|
||||
assert "_custom_mm2" in IrOp.registry
|
||||
assert IrOp.registry["_custom_mm2"] is _custom_mm2
|
||||
assert _custom_mm2.torch_op is torch.ops.vllm_ir._custom_mm2.default
|
||||
assert isinstance(_custom_mm2.maybe_inplace, IrOpInplaceOverload)
|
||||
assert (
|
||||
_custom_mm2.maybe_inplace.torch_op
|
||||
is torch.ops.vllm_ir._custom_mm2.maybe_inplace
|
||||
)
|
||||
|
||||
def test_inplace_dispatching(self):
|
||||
# check that the correct implementation is dispatched based on priority,
|
||||
# and inplace semantics hold
|
||||
w = torch.randn(3, 3)
|
||||
x = torch.randn(2, 3)
|
||||
x1 = x.clone()
|
||||
|
||||
with _custom_mm2.set_priority(["regular"]):
|
||||
result_regular = _custom_mm2.maybe_inplace(x, w)
|
||||
|
||||
# check that the regular op does not modify x
|
||||
torch.testing.assert_close(x, x1, atol=0, rtol=0)
|
||||
|
||||
with _custom_mm2.set_priority(["inplace"]):
|
||||
result_inplace: Tensor = _custom_mm2.maybe_inplace(x, w)
|
||||
|
||||
# check that the inplace op returns x directly
|
||||
assert result_inplace.data_ptr() == x.data_ptr()
|
||||
|
||||
torch.testing.assert_close(result_inplace, x1 @ w + 2)
|
||||
torch.testing.assert_close(result_regular, x1 @ w + 1)
|
||||
|
||||
def test_default_dispatching(self):
|
||||
# check that the correct implementation is dispatched,
|
||||
# and ops do not modify inputs when using the default overload
|
||||
w = torch.randn(3, 3)
|
||||
x = torch.randn(2, 3)
|
||||
x1 = x.clone()
|
||||
|
||||
with _custom_mm2.set_priority(["regular"]):
|
||||
result_regular = _custom_mm2(x, w)
|
||||
|
||||
with _custom_mm2.set_priority(["inplace"]):
|
||||
result_inplace = _custom_mm2(x, w)
|
||||
|
||||
# check that x was not modified by either impl
|
||||
torch.testing.assert_close(x, x1, atol=0, rtol=0)
|
||||
|
||||
torch.testing.assert_close(result_inplace, x1 @ w + 2)
|
||||
torch.testing.assert_close(result_regular, x1 @ w + 1)
|
||||
|
||||
def test_trace(self):
|
||||
# Test that the inplace op can be used in a graph.
|
||||
def func(x: Tensor, y: Tensor) -> Tensor:
|
||||
return _custom_mm2.maybe_inplace(x, y)
|
||||
|
||||
x = torch.randn(2, 3)
|
||||
y = torch.randn(3, 4)
|
||||
graph = make_fx(func)(x, y)
|
||||
assert any(
|
||||
node.target == torch.ops.vllm_ir._custom_mm2.maybe_inplace
|
||||
for node in graph.graph.nodes
|
||||
)
|
||||
+27
-16
@@ -21,7 +21,7 @@ class CustomError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
@vllm.ir.register_op
|
||||
@vllm.ir.register_op(allow_inplace=True)
|
||||
def _custom_add(x: torch.Tensor, y: torch.Tensor) -> torch.Tensor:
|
||||
return x + y
|
||||
|
||||
@@ -129,11 +129,15 @@ class TestIrOpCustomAdd:
|
||||
|
||||
@pytest.mark.parametrize("enable_torch_wrap", [True, False])
|
||||
@pytest.mark.parametrize("symbolic_trace", [True, False])
|
||||
@pytest.mark.parametrize("overload", ["default", "maybe_inplace"])
|
||||
def test_trace_sees_single_custom_op(
|
||||
self, symbolic_trace: bool, enable_torch_wrap: bool
|
||||
self, symbolic_trace: bool, enable_torch_wrap: bool, overload: str
|
||||
):
|
||||
op_fn = _custom_add if overload == "default" else _custom_add.maybe_inplace
|
||||
torch_op = getattr(torch.ops.vllm_ir._custom_add, overload)
|
||||
|
||||
def fn(x, y):
|
||||
return _custom_add(x, y)
|
||||
return op_fn(x, y)
|
||||
|
||||
def find_fn(target: Any, gm: fx.GraphModule):
|
||||
return gm.graph.find_nodes(op="call_function", target=target)
|
||||
@@ -155,7 +159,7 @@ class TestIrOpCustomAdd:
|
||||
torch.testing.assert_close(out_fx, out_eager)
|
||||
|
||||
# check that IR nodes only appear if enable_torch_wrap=True
|
||||
ir_nodes = find_fn(torch.ops.vllm_ir._custom_add.default, gm)
|
||||
ir_nodes = find_fn(torch_op, gm)
|
||||
if enable_torch_wrap:
|
||||
assert len(ir_nodes) == 1, gm.code
|
||||
else:
|
||||
@@ -167,7 +171,7 @@ class TestIrOpCustomAdd:
|
||||
else:
|
||||
gm = make_fx(fn)(torch.randn(2, 2), torch.randn(2, 2))
|
||||
|
||||
ir_nodes = find_fn(torch.ops.vllm_ir._custom_add.default, gm)
|
||||
ir_nodes = find_fn(torch_op, gm)
|
||||
assert len(ir_nodes) == 1, gm.code
|
||||
|
||||
|
||||
@@ -176,9 +180,12 @@ def impl_a(x: torch.Tensor, y: torch.Tensor) -> torch.Tensor:
|
||||
return x + y + 10
|
||||
|
||||
|
||||
@_custom_add.register_impl("impl_b")
|
||||
@_custom_add.register_impl("impl_b", inplace=True)
|
||||
def impl_b(x: torch.Tensor, y: torch.Tensor) -> torch.Tensor:
|
||||
return x + y + 20
|
||||
"""Computes x+y+20"""
|
||||
x.add_(y)
|
||||
x.add_(20)
|
||||
return x
|
||||
|
||||
|
||||
@_custom_add.register_impl("impl_even", supports_args=lambda x, y: x.size(1) % 2 == 0)
|
||||
@@ -243,19 +250,23 @@ class TestIrOpImplDispatch:
|
||||
# Restored to empty
|
||||
assert _custom_add.get_priority() == []
|
||||
|
||||
def test_dispatch_priority_order(self):
|
||||
@pytest.mark.parametrize("overload", ["default", "maybe_inplace"])
|
||||
def test_dispatch_priority_order(self, overload: str):
|
||||
op_fn = _custom_add if overload == "default" else _custom_add.maybe_inplace
|
||||
torch_op = getattr(torch.ops.vllm_ir._custom_add, overload)
|
||||
|
||||
x = torch.tensor(1, dtype=torch.int32)
|
||||
y = torch.tensor(2, dtype=torch.int32)
|
||||
|
||||
with _custom_add.set_priority(["impl_b", "impl_a"]):
|
||||
assert _custom_add.dispatch(x, y) is impl_b
|
||||
out1 = _custom_add(x, y)
|
||||
out2 = torch.ops.vllm_ir._custom_add(x, y)
|
||||
out1 = op_fn(x.clone(), y)
|
||||
out2 = torch_op(x.clone(), y)
|
||||
|
||||
with _custom_add.set_priority(["impl_a"]):
|
||||
assert _custom_add.dispatch(x, y) is impl_a
|
||||
out3 = _custom_add(x, y)
|
||||
out4 = torch.ops.vllm_ir._custom_add(x, y)
|
||||
out3 = op_fn(x.clone(), y)
|
||||
out4 = torch_op(x.clone(), y)
|
||||
|
||||
# impl_b
|
||||
assert out1.item() == 1 + 2 + 20
|
||||
@@ -265,18 +276,18 @@ class TestIrOpImplDispatch:
|
||||
assert out4.item() == 1 + 2 + 10
|
||||
|
||||
def test_unsupported_impl_filtered(self):
|
||||
@_custom_add.register_impl("unsupported", supported=False)
|
||||
def impl_bad(x: torch.Tensor, y: torch.Tensor) -> torch.Tensor:
|
||||
@_custom_add.register_impl("impl_unsupported", supported=False)
|
||||
def impl_unsupported(x: torch.Tensor, y: torch.Tensor) -> torch.Tensor:
|
||||
return x + y + 999
|
||||
|
||||
x = torch.tensor(1, dtype=torch.int32)
|
||||
y = torch.tensor(2, dtype=torch.int32)
|
||||
|
||||
with _custom_add.set_priority(["unsupported", "impl_a"]):
|
||||
with _custom_add.set_priority(["impl_unsupported", "impl_a"]):
|
||||
assert _custom_add.get_priority() == ["impl_a"]
|
||||
out = _custom_add(x, y)
|
||||
|
||||
# impl_bad skipped → impl_a
|
||||
# impl_unsupported skipped → impl_a
|
||||
assert out.item() == 1 + 2 + 10
|
||||
|
||||
def test_supports_args_runtime_dispatch_and_warning(
|
||||
|
||||
@@ -28,7 +28,9 @@ def test_rms_norm_registration():
|
||||
"native": True,
|
||||
"vllm_c": current_platform.is_cuda_alike(),
|
||||
"aiter": current_platform.is_rocm(),
|
||||
"oink": False,
|
||||
"oink": current_platform.has_device_capability(100)
|
||||
and hasattr(torch.ops, "oink")
|
||||
and hasattr(torch.ops.oink, "rmsnorm"),
|
||||
"xpu_kernels": current_platform.is_xpu(),
|
||||
}
|
||||
|
||||
@@ -67,6 +69,14 @@ class TestRMSNorm:
|
||||
out2 = rms_norm_native(x * 2.0, weight, epsilon=epsilon)
|
||||
torch.testing.assert_close(out2, out, rtol=get_default_rtol(out), atol=1e-3)
|
||||
|
||||
# Mean square should be approximately 1 (ignoring epsilon and weight scaling)
|
||||
combined_norm = out.float() / weight.float()
|
||||
variance = combined_norm.pow(2).mean(dim=-1)
|
||||
# After RMS normalization, variance should be close to 1
|
||||
torch.testing.assert_close(
|
||||
variance, torch.ones_like(variance), rtol=1e-2, atol=1e-2
|
||||
)
|
||||
|
||||
# Check behavior with and without weight
|
||||
weight1 = torch.ones_like(weight)
|
||||
out3 = rms_norm_native(x, weight1, epsilon=epsilon)
|
||||
@@ -129,3 +139,197 @@ def test_aiter_rejects_unsupported_dtypes():
|
||||
num_tokens=8, hidden_size=4096, dtype=dtype, epsilon=1e-5
|
||||
)
|
||||
assert not impl.supports_args(*args), f"aiter should reject dtype={dtype}"
|
||||
|
||||
|
||||
fused_add_rms_norm_native = ir.ops.fused_add_rms_norm.impls["native"].impl_fn
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda_alike() and not current_platform.is_xpu(),
|
||||
reason="Currently only kernels on CUDA, ROCm and XPU",
|
||||
)
|
||||
def test_fused_add_rms_norm_registration():
|
||||
expected = {
|
||||
"native": True,
|
||||
"vllm_c": current_platform.is_cuda_alike(),
|
||||
"aiter": current_platform.is_rocm(),
|
||||
"oink": current_platform.has_device_capability(100)
|
||||
and hasattr(torch.ops, "oink")
|
||||
and hasattr(torch.ops.oink, "fused_add_rms_norm"),
|
||||
"xpu_kernels": current_platform.is_xpu(),
|
||||
}
|
||||
|
||||
actual = {
|
||||
provider: impl.supported
|
||||
for provider, impl in ir.ops.fused_add_rms_norm.impls.items()
|
||||
}
|
||||
|
||||
assert actual == expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16, torch.float32])
|
||||
@pytest.mark.parametrize("n_tokens", NUM_TOKENS)
|
||||
@pytest.mark.parametrize("hidden_size", COMMON_HIDDEN_SIZES)
|
||||
@pytest.mark.parametrize("epsilon", [1e-6, 1e-5])
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda_alike() and not current_platform.is_xpu(),
|
||||
reason="Currently only kernels on CUDA, ROCm and XPU",
|
||||
)
|
||||
class TestFusedAddRMSNorm:
|
||||
@classmethod
|
||||
def setup_class(cls, **kwargs):
|
||||
torch.set_default_device(current_platform.device_type)
|
||||
|
||||
def test_native_semantics(self, dtype, n_tokens, hidden_size, epsilon):
|
||||
x, x_residual, weight, eps = ir.ops.fused_add_rms_norm.generate_inputs(
|
||||
num_tokens=4, hidden_size=8, dtype=dtype, epsilon=epsilon
|
||||
)
|
||||
out, residual_out = fused_add_rms_norm_native(x, x_residual, weight, eps)
|
||||
|
||||
# Check shape, dtype, device
|
||||
assert out.shape == x.shape
|
||||
assert out.dtype == x.dtype
|
||||
assert out.device == x.device
|
||||
assert residual_out.shape == x_residual.shape
|
||||
assert residual_out.dtype == x_residual.dtype
|
||||
assert residual_out.device == x_residual.device
|
||||
|
||||
# Check that residual_out = x + x_residual
|
||||
expected_residual = (x.float() + x_residual.float()).to(dtype)
|
||||
torch.testing.assert_close(
|
||||
residual_out, expected_residual, rtol=1e-3, atol=1e-3
|
||||
)
|
||||
|
||||
# Verify that the output is RMS normalized version of (x + x_residual)
|
||||
expected_out = rms_norm_native(expected_residual, weight, epsilon)
|
||||
assert_close(
|
||||
ir.ops.fused_add_rms_norm,
|
||||
(out, residual_out),
|
||||
(expected_out, expected_residual),
|
||||
)
|
||||
|
||||
# Check the scaling property of rms norm
|
||||
out1, _ = fused_add_rms_norm_native(
|
||||
x, torch.zeros_like(x), weight, epsilon=epsilon
|
||||
)
|
||||
out2, _ = fused_add_rms_norm_native(
|
||||
x * 2.0, torch.zeros_like(x), weight, epsilon=epsilon
|
||||
)
|
||||
torch.testing.assert_close(out2, out1, rtol=get_default_rtol(out), atol=1e-3)
|
||||
|
||||
# Check behavior with and without weight
|
||||
weight1 = torch.ones_like(weight)
|
||||
out3, _ = fused_add_rms_norm_native(x, x_residual, weight1, eps)
|
||||
out4, _ = fused_add_rms_norm_native(x, x_residual, None, eps)
|
||||
torch.testing.assert_close(out3, out4)
|
||||
|
||||
@pytest.mark.parametrize("provider", supported_providers(ir.ops.fused_add_rms_norm))
|
||||
def test_impls(self, dtype, n_tokens, hidden_size, epsilon, provider):
|
||||
impl = ir.ops.fused_add_rms_norm.impls[provider]
|
||||
x, x_residual, weight, eps = ir.ops.fused_add_rms_norm.generate_inputs(
|
||||
num_tokens=n_tokens, hidden_size=hidden_size, dtype=dtype, epsilon=epsilon
|
||||
)
|
||||
args = (x, x_residual, weight, eps, None)
|
||||
|
||||
if not impl.supports_args(*args):
|
||||
pytest.skip(f"{provider} does not support args")
|
||||
|
||||
ref_output, ref_residual = fused_add_rms_norm_native(*clone_args(args))
|
||||
output, residual = impl.impl_fn(*clone_args(args))
|
||||
assert_close(ir.ops.fused_add_rms_norm, output, ref_output)
|
||||
assert_close(ir.ops.fused_add_rms_norm, residual, ref_residual)
|
||||
|
||||
# check that dispatched call matches direct call
|
||||
with ir.ops.fused_add_rms_norm.set_priority([provider, "native"]):
|
||||
out_dispatched, residual_dispatched = ir.ops.fused_add_rms_norm(*args[:4])
|
||||
out_direct, residual_direct = impl.impl_fn(*clone_args(args))
|
||||
torch.testing.assert_close(out_dispatched, out_direct, rtol=0.0, atol=0.0)
|
||||
torch.testing.assert_close(
|
||||
residual_dispatched, residual_direct, rtol=0.0, atol=0.0
|
||||
)
|
||||
|
||||
# none of these support variance_size override
|
||||
assert not impl.supports_args(x, x_residual, weight, epsilon, 4)
|
||||
assert not impl.supports_args(x, x_residual, weight, epsilon, variance_size=4)
|
||||
|
||||
# test weight=None behavior
|
||||
out_no_weight, residual_no_weight = impl.impl_fn(
|
||||
x.clone(), x_residual.clone(), None, epsilon
|
||||
)
|
||||
out_unit_weight, residual_unit_weight = impl.impl_fn(
|
||||
x.clone(), x_residual.clone(), torch.ones_like(weight), epsilon
|
||||
)
|
||||
assert_close(ir.ops.fused_add_rms_norm, out_no_weight, out_unit_weight)
|
||||
assert_close(
|
||||
ir.ops.fused_add_rms_norm, residual_no_weight, residual_unit_weight
|
||||
)
|
||||
|
||||
@pytest.mark.parametrize("provider", ["vllm_c"])
|
||||
def test_inplace_semantics(self, dtype, n_tokens, hidden_size, epsilon, provider):
|
||||
"""Test that inplace implementations reuse inputs,
|
||||
for maybe_inplace overload but not for default overload."""
|
||||
impl = ir.ops.fused_add_rms_norm.impls[provider]
|
||||
if not impl.supported:
|
||||
pytest.skip(f"{provider} impl not supported on this platform")
|
||||
|
||||
x, x_residual, weight, eps = ir.ops.fused_add_rms_norm.generate_inputs(
|
||||
num_tokens=n_tokens, hidden_size=hidden_size, dtype=dtype, epsilon=epsilon
|
||||
)
|
||||
|
||||
# Test default overload - should NOT modify inputs even with inplace impl
|
||||
x_default = x.clone()
|
||||
x_residual_default = x_residual.clone()
|
||||
x_default_ptr = x_default.data_ptr()
|
||||
x_residual_default_ptr = x_residual_default.data_ptr()
|
||||
|
||||
with ir.ops.fused_add_rms_norm.set_priority([provider, "native"]):
|
||||
out_default, residual_default = ir.ops.fused_add_rms_norm(
|
||||
x_default, x_residual_default, weight, eps
|
||||
)
|
||||
|
||||
# Default should NOT be inplace (even with inplace implementation)
|
||||
assert out_default.data_ptr() != x_default_ptr
|
||||
assert residual_default.data_ptr() != x_residual_default_ptr
|
||||
torch.testing.assert_close(x, x_default, rtol=0.0, atol=0.0)
|
||||
torch.testing.assert_close(x_residual, x_residual_default, rtol=0.0, atol=0.0)
|
||||
|
||||
# Test maybe_inplace overload - should modify inputs with inplace impl
|
||||
x_inplace = x.clone()
|
||||
x_residual_inplace = x_residual.clone()
|
||||
x_inplace_ptr = x_inplace.data_ptr()
|
||||
x_residual_inplace_ptr = x_residual_inplace.data_ptr()
|
||||
|
||||
with ir.ops.fused_add_rms_norm.set_priority([provider, "native"]):
|
||||
out_inplace, residual_inplace = ir.ops.fused_add_rms_norm.maybe_inplace(
|
||||
x_inplace, x_residual_inplace, weight, eps
|
||||
)
|
||||
|
||||
# maybe_inplace should be inplace
|
||||
assert out_inplace.data_ptr() == x_inplace_ptr
|
||||
assert residual_inplace.data_ptr() == x_residual_inplace_ptr
|
||||
|
||||
# Both should produce same results
|
||||
torch.testing.assert_close(out_default, out_inplace, atol=0.0, rtol=0.0)
|
||||
torch.testing.assert_close(
|
||||
residual_default, residual_inplace, atol=0.0, rtol=0.0
|
||||
)
|
||||
|
||||
@pytest.mark.parametrize("provider", supported_providers(ir.ops.fused_add_rms_norm))
|
||||
def test_torch_opcheck(self, dtype, n_tokens, hidden_size, epsilon, provider):
|
||||
args = ir.ops.fused_add_rms_norm.generate_inputs(
|
||||
num_tokens=n_tokens, hidden_size=hidden_size, dtype=dtype, epsilon=epsilon
|
||||
)
|
||||
args = args + (None,) # Add variance_size parameter
|
||||
|
||||
# When checking the torch op, we have to set priority and use dispatch
|
||||
with ir.ops.fused_add_rms_norm.set_priority([provider, "native"]):
|
||||
torch.library.opcheck(torch.ops.vllm_ir.fused_add_rms_norm.default, args)
|
||||
|
||||
# Only test maybe_inplace with non-inplace implementations
|
||||
# Inplace implementations return aliases of inputs which is not allowed.
|
||||
# We break this invariant, but we also convert maybe_inplace to the default
|
||||
# overload during compilation, so maybe_inplace never reaches Inductor.
|
||||
if not ir.ops.fused_add_rms_norm.impls[provider].inplace:
|
||||
torch.library.opcheck(
|
||||
torch.ops.vllm_ir.fused_add_rms_norm.maybe_inplace, args
|
||||
)
|
||||
|
||||
@@ -23,11 +23,7 @@ from vllm.model_executor.layers.fused_moe.router.fused_topk_router import (
|
||||
vllm_topk_sigmoid,
|
||||
vllm_topk_softmax,
|
||||
)
|
||||
from vllm.model_executor.layers.layernorm import (
|
||||
RMSNorm,
|
||||
dispatch_rocm_rmsnorm_func,
|
||||
fused_add_rms_norm,
|
||||
)
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
RMS_NORM_SUPPORTED_DTYPES = [torch.float16, torch.bfloat16]
|
||||
@@ -153,26 +149,3 @@ def test_topk_sigmoid_dispatch(use_rocm_aiter: bool):
|
||||
assert topk_func == rocm_aiter_ops.topk_sigmoid
|
||||
else:
|
||||
assert topk_func == vllm_topk_sigmoid
|
||||
|
||||
|
||||
@pytest.mark.parametrize("add_residual", [False])
|
||||
@pytest.mark.parametrize("dtype", [torch.float32, torch.float16, torch.bfloat16])
|
||||
@pytest.mark.parametrize("use_rocm_aiter", [True, False])
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_rocm(), reason="AITER is a feature exclusive for ROCm"
|
||||
)
|
||||
def test_rms_norm_dispatch(
|
||||
add_residual: bool, dtype: torch.dtype, use_rocm_aiter: bool
|
||||
):
|
||||
rms_norm_func = dispatch_rocm_rmsnorm_func(dtype, use_rocm_aiter)
|
||||
|
||||
should_use_rocm_aiter = (
|
||||
current_platform.is_rocm()
|
||||
and use_rocm_aiter
|
||||
and dtype in RMS_NORM_SUPPORTED_DTYPES
|
||||
)
|
||||
|
||||
if should_use_rocm_aiter:
|
||||
assert rms_norm_func == rocm_aiter_ops.rms_norm2d_with_add
|
||||
else:
|
||||
assert rms_norm_func == fused_add_rms_norm
|
||||
|
||||
@@ -1,60 +1,97 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import multiprocessing
|
||||
import types
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
|
||||
def _load_oink_ops_module():
|
||||
# Import the module normally (vllm is installed as an editable package in CI).
|
||||
from vllm import _oink_ops
|
||||
def _test_oink_availability_impl(
|
||||
device_capability: tuple[int, int],
|
||||
has_rmsnorm: bool,
|
||||
has_fused_add_rms_norm: bool,
|
||||
expected_available: bool,
|
||||
expected_fused: bool,
|
||||
) -> None:
|
||||
"""Test OINK support detection with mocked state."""
|
||||
import torch
|
||||
|
||||
return _oink_ops
|
||||
from vllm import platforms
|
||||
|
||||
# Mock device capability (class method, override on class)
|
||||
dc = platforms.interface.DeviceCapability(*device_capability)
|
||||
platforms.current_platform.__class__.get_device_capability = lambda device_id=0: dc
|
||||
|
||||
# Mock oink ops
|
||||
oink_ops = types.SimpleNamespace()
|
||||
if has_rmsnorm:
|
||||
oink_ops.rmsnorm = lambda x, w, eps: x
|
||||
if has_fused_add_rms_norm:
|
||||
oink_ops.fused_add_rms_norm = lambda x, residual, w, eps: None
|
||||
|
||||
torch.ops.oink = oink_ops
|
||||
|
||||
# Now import vllm modules with mocks in place (fresh import with mocked platform)
|
||||
import vllm.kernels.oink_ops # noqa: F401
|
||||
from vllm.ir.ops import fused_add_rms_norm, rms_norm
|
||||
|
||||
# Verify support checks
|
||||
assert rms_norm.impls["oink"].supported is expected_available
|
||||
assert fused_add_rms_norm.impls["oink"].supported is expected_fused
|
||||
|
||||
|
||||
def test_oink_availability_checks(monkeypatch: pytest.MonkeyPatch):
|
||||
_oink_ops = _load_oink_ops_module()
|
||||
@pytest.mark.parametrize(
|
||||
"device_capability,has_rmsnorm,has_fused_add_rms_norm,expected_available,expected_fused",
|
||||
[
|
||||
# Case 1: < SM100, ops not supported
|
||||
((9, 0), True, False, False, False),
|
||||
# Case 2: CUDA available and SM100, rmsnorm op registered
|
||||
((10, 0), True, False, True, False),
|
||||
# Case 3: SM100 with both rmsnorm and fused_add_rms_norm
|
||||
((10, 0), True, True, True, True),
|
||||
],
|
||||
)
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="Only test on CUDA")
|
||||
def test_oink_availability_checks(
|
||||
device_capability: tuple[int, int],
|
||||
has_rmsnorm: bool,
|
||||
has_fused_add_rms_norm: bool,
|
||||
expected_available: bool,
|
||||
expected_fused: bool,
|
||||
):
|
||||
"""Test OINK support detection with clean import state for each parameter set."""
|
||||
|
||||
# Ensure the ops namespace exists and is mutable for tests.
|
||||
monkeypatch.setattr(
|
||||
torch.ops,
|
||||
"oink",
|
||||
types.SimpleNamespace(rmsnorm=lambda x, w, eps: x),
|
||||
raising=False,
|
||||
)
|
||||
|
||||
# Case 1: CUDA not available.
|
||||
monkeypatch.setattr(torch.cuda, "is_available", lambda: False)
|
||||
assert _oink_ops.is_oink_available_for_device(0) is False
|
||||
|
||||
# Case 2: CUDA available but < SM100.
|
||||
monkeypatch.setattr(torch.cuda, "is_available", lambda: True)
|
||||
monkeypatch.setattr(torch.cuda, "get_device_capability", lambda idx: (9, 0))
|
||||
assert _oink_ops.is_oink_available_for_device(0) is False
|
||||
|
||||
# Case 3: CUDA available and SM100, rmsnorm op registered.
|
||||
monkeypatch.setattr(torch.cuda, "get_device_capability", lambda idx: (10, 0))
|
||||
assert _oink_ops.is_oink_available_for_device(0) is True
|
||||
|
||||
# fused op presence probe
|
||||
assert _oink_ops.has_fused_add_rms_norm() is False
|
||||
monkeypatch.setattr(
|
||||
torch.ops,
|
||||
"oink",
|
||||
types.SimpleNamespace(
|
||||
rmsnorm=lambda x, w, eps: x,
|
||||
fused_add_rms_norm=lambda x, residual, w, eps: None,
|
||||
# Use spawn to run function in fresh process with clean imports
|
||||
# TODO migrate to spawn utility:
|
||||
# https://github.com/vllm-project/vllm/issues/41415
|
||||
ctx = multiprocessing.get_context("spawn")
|
||||
process = ctx.Process(
|
||||
target=_test_oink_availability_impl,
|
||||
args=(
|
||||
device_capability,
|
||||
has_rmsnorm,
|
||||
has_fused_add_rms_norm,
|
||||
expected_available,
|
||||
expected_fused,
|
||||
),
|
||||
raising=False,
|
||||
)
|
||||
assert _oink_ops.has_fused_add_rms_norm() is True
|
||||
process.start()
|
||||
process.join()
|
||||
|
||||
if process.exitcode != 0:
|
||||
raise AssertionError(
|
||||
f"Subprocess test failed with exit code {process.exitcode}"
|
||||
)
|
||||
|
||||
|
||||
def test_can_view_as_2d_stride_guard():
|
||||
# Import the helper from the layernorm module.
|
||||
from vllm.model_executor.layers.layernorm import _can_view_as_2d
|
||||
# No global import
|
||||
import torch
|
||||
|
||||
# Import the helper from the kernels module.
|
||||
from vllm.kernels.oink_ops import _can_view_as_2d
|
||||
|
||||
x = torch.zeros((2, 3, 4))
|
||||
assert _can_view_as_2d(x) is True
|
||||
|
||||
+37
-3
@@ -1295,11 +1295,14 @@ def test_ir_op_priority_default():
|
||||
# Assert default is applied to ops
|
||||
priority_config = IrOpPriorityConfig.with_default(["vllm_c", "native"])
|
||||
assert priority_config.rms_norm == ["vllm_c", "native"]
|
||||
assert priority_config.fused_add_rms_norm == ["vllm_c", "native"]
|
||||
|
||||
# Assert single ops override the default
|
||||
assert IrOpPriorityConfig.with_default(
|
||||
["vllm_c", "native"], rms_norm=["oink", "native"]
|
||||
) == IrOpPriorityConfig(rms_norm=["oink", "native"])
|
||||
priority_config = IrOpPriorityConfig.with_default(
|
||||
["native"], rms_norm=["oink", "native"]
|
||||
)
|
||||
assert priority_config.rms_norm == ["oink", "native"]
|
||||
assert priority_config.fused_add_rms_norm == ["native"]
|
||||
|
||||
|
||||
def test_ir_op_priority_str():
|
||||
@@ -1318,3 +1321,34 @@ def test_ir_op_priority_str():
|
||||
with pytest.raises(pydantic.ValidationError):
|
||||
# must be list of only strings
|
||||
priority_config = IrOpPriorityConfig(rms_norm=["vllm_c", 4, "native"])
|
||||
|
||||
|
||||
def test_ir_op_priority_ctx():
|
||||
"""Test that the priority-setting context sets priority correctly."""
|
||||
from vllm import ir
|
||||
from vllm.config.kernel import IrOpPriorityConfig
|
||||
|
||||
priority = IrOpPriorityConfig.with_default(["native"], rms_norm=["vllm_c"])
|
||||
priority2 = IrOpPriorityConfig.with_default(
|
||||
["native"], fused_add_rms_norm=["vllm_c"]
|
||||
)
|
||||
with priority.set_priority():
|
||||
assert ir.ops.rms_norm.get_priority() == ["vllm_c", "native"]
|
||||
assert ir.ops.fused_add_rms_norm.get_priority() == ["native"]
|
||||
with priority2.set_priority():
|
||||
assert ir.ops.rms_norm.get_priority() == ["native"]
|
||||
assert ir.ops.fused_add_rms_norm.get_priority() == ["vllm_c", "native"]
|
||||
|
||||
# context restored
|
||||
assert ir.ops.rms_norm.get_priority() == ["vllm_c", "native"]
|
||||
assert ir.ops.fused_add_rms_norm.get_priority() == ["native"]
|
||||
|
||||
with pytest.raises(ValueError), priority2.set_priority():
|
||||
assert ir.ops.rms_norm.get_priority() == ["native"]
|
||||
assert ir.ops.fused_add_rms_norm.get_priority() == ["vllm_c", "native"]
|
||||
|
||||
raise ValueError
|
||||
|
||||
# context restored even after exception
|
||||
assert ir.ops.rms_norm.get_priority() == ["vllm_c", "native"]
|
||||
assert ir.ops.fused_add_rms_norm.get_priority() == ["native"]
|
||||
|
||||
@@ -12,7 +12,7 @@ import torch
|
||||
from utils import skip_unsupported
|
||||
|
||||
from vllm.model_executor.layers.batch_invariant import rms_norm as triton_rms_norm
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm, fused_add_rms_norm
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
DEVICE_TYPE = current_platform.device_type
|
||||
@@ -105,6 +105,12 @@ def test_fused_add_rms_norm_batch_invariant_residual_path(
|
||||
dim=0,
|
||||
)
|
||||
|
||||
def fused_add_rms_norm(x, residual, w, e) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
import vllm._custom_ops as ops
|
||||
|
||||
ops.fused_add_rms_norm(x, residual, w, e)
|
||||
return x, residual
|
||||
|
||||
out_single, residual_out_single = fused_add_rms_norm(
|
||||
x_single.clone(),
|
||||
residual_single.clone(),
|
||||
|
||||
Reference in New Issue
Block a user