[vLLM IR] 2/N fused_add_rms_norm and maybe_inplace overload (#36823)

Signed-off-by: Luka Govedič <lgovedic@redhat.com>
Signed-off-by: Luka Govedič <ProExpertProg@users.noreply.github.com>
This commit is contained in:
Luka Govedič
2026-05-01 23:41:15 -04:00
committed by GitHub
parent 3e49479c4b
commit d58c42e19c
44 changed files with 2833 additions and 779 deletions
+91
View File
@@ -0,0 +1,91 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
from torch import Tensor
from torch.fx.experimental.proxy_tensor import make_fx
import vllm.ir.op
from vllm.ir.op import IrOp, IrOpInplaceOverload
@vllm.ir.register_op(allow_inplace=True)
def _custom_mm2(x: Tensor, w: Tensor) -> Tensor:
return x @ w
@_custom_mm2.register_impl("regular")
def _custom_mm2_regular(x: Tensor, w: Tensor) -> Tensor:
return x @ w + 1
@_custom_mm2.register_impl("inplace", inplace=True)
def _custom_mm2_inplace(x: Tensor, w: Tensor) -> Tensor:
x.copy_(x @ w + 2)
return x
class TestInplaceOp:
def test_registration(self):
# Test that the inplace op is registered correctly.
assert "_custom_mm2" in IrOp.registry
assert IrOp.registry["_custom_mm2"] is _custom_mm2
assert _custom_mm2.torch_op is torch.ops.vllm_ir._custom_mm2.default
assert isinstance(_custom_mm2.maybe_inplace, IrOpInplaceOverload)
assert (
_custom_mm2.maybe_inplace.torch_op
is torch.ops.vllm_ir._custom_mm2.maybe_inplace
)
def test_inplace_dispatching(self):
# check that the correct implementation is dispatched based on priority,
# and inplace semantics hold
w = torch.randn(3, 3)
x = torch.randn(2, 3)
x1 = x.clone()
with _custom_mm2.set_priority(["regular"]):
result_regular = _custom_mm2.maybe_inplace(x, w)
# check that the regular op does not modify x
torch.testing.assert_close(x, x1, atol=0, rtol=0)
with _custom_mm2.set_priority(["inplace"]):
result_inplace: Tensor = _custom_mm2.maybe_inplace(x, w)
# check that the inplace op returns x directly
assert result_inplace.data_ptr() == x.data_ptr()
torch.testing.assert_close(result_inplace, x1 @ w + 2)
torch.testing.assert_close(result_regular, x1 @ w + 1)
def test_default_dispatching(self):
# check that the correct implementation is dispatched,
# and ops do not modify inputs when using the default overload
w = torch.randn(3, 3)
x = torch.randn(2, 3)
x1 = x.clone()
with _custom_mm2.set_priority(["regular"]):
result_regular = _custom_mm2(x, w)
with _custom_mm2.set_priority(["inplace"]):
result_inplace = _custom_mm2(x, w)
# check that x was not modified by either impl
torch.testing.assert_close(x, x1, atol=0, rtol=0)
torch.testing.assert_close(result_inplace, x1 @ w + 2)
torch.testing.assert_close(result_regular, x1 @ w + 1)
def test_trace(self):
# Test that the inplace op can be used in a graph.
def func(x: Tensor, y: Tensor) -> Tensor:
return _custom_mm2.maybe_inplace(x, y)
x = torch.randn(2, 3)
y = torch.randn(3, 4)
graph = make_fx(func)(x, y)
assert any(
node.target == torch.ops.vllm_ir._custom_mm2.maybe_inplace
for node in graph.graph.nodes
)
+27 -16
View File
@@ -21,7 +21,7 @@ class CustomError(Exception):
pass
@vllm.ir.register_op
@vllm.ir.register_op(allow_inplace=True)
def _custom_add(x: torch.Tensor, y: torch.Tensor) -> torch.Tensor:
return x + y
@@ -129,11 +129,15 @@ class TestIrOpCustomAdd:
@pytest.mark.parametrize("enable_torch_wrap", [True, False])
@pytest.mark.parametrize("symbolic_trace", [True, False])
@pytest.mark.parametrize("overload", ["default", "maybe_inplace"])
def test_trace_sees_single_custom_op(
self, symbolic_trace: bool, enable_torch_wrap: bool
self, symbolic_trace: bool, enable_torch_wrap: bool, overload: str
):
op_fn = _custom_add if overload == "default" else _custom_add.maybe_inplace
torch_op = getattr(torch.ops.vllm_ir._custom_add, overload)
def fn(x, y):
return _custom_add(x, y)
return op_fn(x, y)
def find_fn(target: Any, gm: fx.GraphModule):
return gm.graph.find_nodes(op="call_function", target=target)
@@ -155,7 +159,7 @@ class TestIrOpCustomAdd:
torch.testing.assert_close(out_fx, out_eager)
# check that IR nodes only appear if enable_torch_wrap=True
ir_nodes = find_fn(torch.ops.vllm_ir._custom_add.default, gm)
ir_nodes = find_fn(torch_op, gm)
if enable_torch_wrap:
assert len(ir_nodes) == 1, gm.code
else:
@@ -167,7 +171,7 @@ class TestIrOpCustomAdd:
else:
gm = make_fx(fn)(torch.randn(2, 2), torch.randn(2, 2))
ir_nodes = find_fn(torch.ops.vllm_ir._custom_add.default, gm)
ir_nodes = find_fn(torch_op, gm)
assert len(ir_nodes) == 1, gm.code
@@ -176,9 +180,12 @@ def impl_a(x: torch.Tensor, y: torch.Tensor) -> torch.Tensor:
return x + y + 10
@_custom_add.register_impl("impl_b")
@_custom_add.register_impl("impl_b", inplace=True)
def impl_b(x: torch.Tensor, y: torch.Tensor) -> torch.Tensor:
return x + y + 20
"""Computes x+y+20"""
x.add_(y)
x.add_(20)
return x
@_custom_add.register_impl("impl_even", supports_args=lambda x, y: x.size(1) % 2 == 0)
@@ -243,19 +250,23 @@ class TestIrOpImplDispatch:
# Restored to empty
assert _custom_add.get_priority() == []
def test_dispatch_priority_order(self):
@pytest.mark.parametrize("overload", ["default", "maybe_inplace"])
def test_dispatch_priority_order(self, overload: str):
op_fn = _custom_add if overload == "default" else _custom_add.maybe_inplace
torch_op = getattr(torch.ops.vllm_ir._custom_add, overload)
x = torch.tensor(1, dtype=torch.int32)
y = torch.tensor(2, dtype=torch.int32)
with _custom_add.set_priority(["impl_b", "impl_a"]):
assert _custom_add.dispatch(x, y) is impl_b
out1 = _custom_add(x, y)
out2 = torch.ops.vllm_ir._custom_add(x, y)
out1 = op_fn(x.clone(), y)
out2 = torch_op(x.clone(), y)
with _custom_add.set_priority(["impl_a"]):
assert _custom_add.dispatch(x, y) is impl_a
out3 = _custom_add(x, y)
out4 = torch.ops.vllm_ir._custom_add(x, y)
out3 = op_fn(x.clone(), y)
out4 = torch_op(x.clone(), y)
# impl_b
assert out1.item() == 1 + 2 + 20
@@ -265,18 +276,18 @@ class TestIrOpImplDispatch:
assert out4.item() == 1 + 2 + 10
def test_unsupported_impl_filtered(self):
@_custom_add.register_impl("unsupported", supported=False)
def impl_bad(x: torch.Tensor, y: torch.Tensor) -> torch.Tensor:
@_custom_add.register_impl("impl_unsupported", supported=False)
def impl_unsupported(x: torch.Tensor, y: torch.Tensor) -> torch.Tensor:
return x + y + 999
x = torch.tensor(1, dtype=torch.int32)
y = torch.tensor(2, dtype=torch.int32)
with _custom_add.set_priority(["unsupported", "impl_a"]):
with _custom_add.set_priority(["impl_unsupported", "impl_a"]):
assert _custom_add.get_priority() == ["impl_a"]
out = _custom_add(x, y)
# impl_bad skipped → impl_a
# impl_unsupported skipped → impl_a
assert out.item() == 1 + 2 + 10
def test_supports_args_runtime_dispatch_and_warning(