[vLLM IR] 2/N fused_add_rms_norm and maybe_inplace overload (#36823)
Signed-off-by: Luka Govedič <lgovedic@redhat.com> Signed-off-by: Luka Govedič <ProExpertProg@users.noreply.github.com>
This commit is contained in:
@@ -0,0 +1,91 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import torch
|
||||
from torch import Tensor
|
||||
from torch.fx.experimental.proxy_tensor import make_fx
|
||||
|
||||
import vllm.ir.op
|
||||
from vllm.ir.op import IrOp, IrOpInplaceOverload
|
||||
|
||||
|
||||
@vllm.ir.register_op(allow_inplace=True)
|
||||
def _custom_mm2(x: Tensor, w: Tensor) -> Tensor:
|
||||
return x @ w
|
||||
|
||||
|
||||
@_custom_mm2.register_impl("regular")
|
||||
def _custom_mm2_regular(x: Tensor, w: Tensor) -> Tensor:
|
||||
return x @ w + 1
|
||||
|
||||
|
||||
@_custom_mm2.register_impl("inplace", inplace=True)
|
||||
def _custom_mm2_inplace(x: Tensor, w: Tensor) -> Tensor:
|
||||
x.copy_(x @ w + 2)
|
||||
return x
|
||||
|
||||
|
||||
class TestInplaceOp:
|
||||
def test_registration(self):
|
||||
# Test that the inplace op is registered correctly.
|
||||
assert "_custom_mm2" in IrOp.registry
|
||||
assert IrOp.registry["_custom_mm2"] is _custom_mm2
|
||||
assert _custom_mm2.torch_op is torch.ops.vllm_ir._custom_mm2.default
|
||||
assert isinstance(_custom_mm2.maybe_inplace, IrOpInplaceOverload)
|
||||
assert (
|
||||
_custom_mm2.maybe_inplace.torch_op
|
||||
is torch.ops.vllm_ir._custom_mm2.maybe_inplace
|
||||
)
|
||||
|
||||
def test_inplace_dispatching(self):
|
||||
# check that the correct implementation is dispatched based on priority,
|
||||
# and inplace semantics hold
|
||||
w = torch.randn(3, 3)
|
||||
x = torch.randn(2, 3)
|
||||
x1 = x.clone()
|
||||
|
||||
with _custom_mm2.set_priority(["regular"]):
|
||||
result_regular = _custom_mm2.maybe_inplace(x, w)
|
||||
|
||||
# check that the regular op does not modify x
|
||||
torch.testing.assert_close(x, x1, atol=0, rtol=0)
|
||||
|
||||
with _custom_mm2.set_priority(["inplace"]):
|
||||
result_inplace: Tensor = _custom_mm2.maybe_inplace(x, w)
|
||||
|
||||
# check that the inplace op returns x directly
|
||||
assert result_inplace.data_ptr() == x.data_ptr()
|
||||
|
||||
torch.testing.assert_close(result_inplace, x1 @ w + 2)
|
||||
torch.testing.assert_close(result_regular, x1 @ w + 1)
|
||||
|
||||
def test_default_dispatching(self):
|
||||
# check that the correct implementation is dispatched,
|
||||
# and ops do not modify inputs when using the default overload
|
||||
w = torch.randn(3, 3)
|
||||
x = torch.randn(2, 3)
|
||||
x1 = x.clone()
|
||||
|
||||
with _custom_mm2.set_priority(["regular"]):
|
||||
result_regular = _custom_mm2(x, w)
|
||||
|
||||
with _custom_mm2.set_priority(["inplace"]):
|
||||
result_inplace = _custom_mm2(x, w)
|
||||
|
||||
# check that x was not modified by either impl
|
||||
torch.testing.assert_close(x, x1, atol=0, rtol=0)
|
||||
|
||||
torch.testing.assert_close(result_inplace, x1 @ w + 2)
|
||||
torch.testing.assert_close(result_regular, x1 @ w + 1)
|
||||
|
||||
def test_trace(self):
|
||||
# Test that the inplace op can be used in a graph.
|
||||
def func(x: Tensor, y: Tensor) -> Tensor:
|
||||
return _custom_mm2.maybe_inplace(x, y)
|
||||
|
||||
x = torch.randn(2, 3)
|
||||
y = torch.randn(3, 4)
|
||||
graph = make_fx(func)(x, y)
|
||||
assert any(
|
||||
node.target == torch.ops.vllm_ir._custom_mm2.maybe_inplace
|
||||
for node in graph.graph.nodes
|
||||
)
|
||||
+27
-16
@@ -21,7 +21,7 @@ class CustomError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
@vllm.ir.register_op
|
||||
@vllm.ir.register_op(allow_inplace=True)
|
||||
def _custom_add(x: torch.Tensor, y: torch.Tensor) -> torch.Tensor:
|
||||
return x + y
|
||||
|
||||
@@ -129,11 +129,15 @@ class TestIrOpCustomAdd:
|
||||
|
||||
@pytest.mark.parametrize("enable_torch_wrap", [True, False])
|
||||
@pytest.mark.parametrize("symbolic_trace", [True, False])
|
||||
@pytest.mark.parametrize("overload", ["default", "maybe_inplace"])
|
||||
def test_trace_sees_single_custom_op(
|
||||
self, symbolic_trace: bool, enable_torch_wrap: bool
|
||||
self, symbolic_trace: bool, enable_torch_wrap: bool, overload: str
|
||||
):
|
||||
op_fn = _custom_add if overload == "default" else _custom_add.maybe_inplace
|
||||
torch_op = getattr(torch.ops.vllm_ir._custom_add, overload)
|
||||
|
||||
def fn(x, y):
|
||||
return _custom_add(x, y)
|
||||
return op_fn(x, y)
|
||||
|
||||
def find_fn(target: Any, gm: fx.GraphModule):
|
||||
return gm.graph.find_nodes(op="call_function", target=target)
|
||||
@@ -155,7 +159,7 @@ class TestIrOpCustomAdd:
|
||||
torch.testing.assert_close(out_fx, out_eager)
|
||||
|
||||
# check that IR nodes only appear if enable_torch_wrap=True
|
||||
ir_nodes = find_fn(torch.ops.vllm_ir._custom_add.default, gm)
|
||||
ir_nodes = find_fn(torch_op, gm)
|
||||
if enable_torch_wrap:
|
||||
assert len(ir_nodes) == 1, gm.code
|
||||
else:
|
||||
@@ -167,7 +171,7 @@ class TestIrOpCustomAdd:
|
||||
else:
|
||||
gm = make_fx(fn)(torch.randn(2, 2), torch.randn(2, 2))
|
||||
|
||||
ir_nodes = find_fn(torch.ops.vllm_ir._custom_add.default, gm)
|
||||
ir_nodes = find_fn(torch_op, gm)
|
||||
assert len(ir_nodes) == 1, gm.code
|
||||
|
||||
|
||||
@@ -176,9 +180,12 @@ def impl_a(x: torch.Tensor, y: torch.Tensor) -> torch.Tensor:
|
||||
return x + y + 10
|
||||
|
||||
|
||||
@_custom_add.register_impl("impl_b")
|
||||
@_custom_add.register_impl("impl_b", inplace=True)
|
||||
def impl_b(x: torch.Tensor, y: torch.Tensor) -> torch.Tensor:
|
||||
return x + y + 20
|
||||
"""Computes x+y+20"""
|
||||
x.add_(y)
|
||||
x.add_(20)
|
||||
return x
|
||||
|
||||
|
||||
@_custom_add.register_impl("impl_even", supports_args=lambda x, y: x.size(1) % 2 == 0)
|
||||
@@ -243,19 +250,23 @@ class TestIrOpImplDispatch:
|
||||
# Restored to empty
|
||||
assert _custom_add.get_priority() == []
|
||||
|
||||
def test_dispatch_priority_order(self):
|
||||
@pytest.mark.parametrize("overload", ["default", "maybe_inplace"])
|
||||
def test_dispatch_priority_order(self, overload: str):
|
||||
op_fn = _custom_add if overload == "default" else _custom_add.maybe_inplace
|
||||
torch_op = getattr(torch.ops.vllm_ir._custom_add, overload)
|
||||
|
||||
x = torch.tensor(1, dtype=torch.int32)
|
||||
y = torch.tensor(2, dtype=torch.int32)
|
||||
|
||||
with _custom_add.set_priority(["impl_b", "impl_a"]):
|
||||
assert _custom_add.dispatch(x, y) is impl_b
|
||||
out1 = _custom_add(x, y)
|
||||
out2 = torch.ops.vllm_ir._custom_add(x, y)
|
||||
out1 = op_fn(x.clone(), y)
|
||||
out2 = torch_op(x.clone(), y)
|
||||
|
||||
with _custom_add.set_priority(["impl_a"]):
|
||||
assert _custom_add.dispatch(x, y) is impl_a
|
||||
out3 = _custom_add(x, y)
|
||||
out4 = torch.ops.vllm_ir._custom_add(x, y)
|
||||
out3 = op_fn(x.clone(), y)
|
||||
out4 = torch_op(x.clone(), y)
|
||||
|
||||
# impl_b
|
||||
assert out1.item() == 1 + 2 + 20
|
||||
@@ -265,18 +276,18 @@ class TestIrOpImplDispatch:
|
||||
assert out4.item() == 1 + 2 + 10
|
||||
|
||||
def test_unsupported_impl_filtered(self):
|
||||
@_custom_add.register_impl("unsupported", supported=False)
|
||||
def impl_bad(x: torch.Tensor, y: torch.Tensor) -> torch.Tensor:
|
||||
@_custom_add.register_impl("impl_unsupported", supported=False)
|
||||
def impl_unsupported(x: torch.Tensor, y: torch.Tensor) -> torch.Tensor:
|
||||
return x + y + 999
|
||||
|
||||
x = torch.tensor(1, dtype=torch.int32)
|
||||
y = torch.tensor(2, dtype=torch.int32)
|
||||
|
||||
with _custom_add.set_priority(["unsupported", "impl_a"]):
|
||||
with _custom_add.set_priority(["impl_unsupported", "impl_a"]):
|
||||
assert _custom_add.get_priority() == ["impl_a"]
|
||||
out = _custom_add(x, y)
|
||||
|
||||
# impl_bad skipped → impl_a
|
||||
# impl_unsupported skipped → impl_a
|
||||
assert out.item() == 1 + 2 + 10
|
||||
|
||||
def test_supports_args_runtime_dispatch_and_warning(
|
||||
|
||||
Reference in New Issue
Block a user