[vLLM IR] 2/N fused_add_rms_norm and maybe_inplace overload (#36823)
Signed-off-by: Luka Govedič <lgovedic@redhat.com> Signed-off-by: Luka Govedič <ProExpertProg@users.noreply.github.com>
This commit is contained in:
+27
-16
@@ -21,7 +21,7 @@ class CustomError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
@vllm.ir.register_op
|
||||
@vllm.ir.register_op(allow_inplace=True)
|
||||
def _custom_add(x: torch.Tensor, y: torch.Tensor) -> torch.Tensor:
|
||||
return x + y
|
||||
|
||||
@@ -129,11 +129,15 @@ class TestIrOpCustomAdd:
|
||||
|
||||
@pytest.mark.parametrize("enable_torch_wrap", [True, False])
|
||||
@pytest.mark.parametrize("symbolic_trace", [True, False])
|
||||
@pytest.mark.parametrize("overload", ["default", "maybe_inplace"])
|
||||
def test_trace_sees_single_custom_op(
|
||||
self, symbolic_trace: bool, enable_torch_wrap: bool
|
||||
self, symbolic_trace: bool, enable_torch_wrap: bool, overload: str
|
||||
):
|
||||
op_fn = _custom_add if overload == "default" else _custom_add.maybe_inplace
|
||||
torch_op = getattr(torch.ops.vllm_ir._custom_add, overload)
|
||||
|
||||
def fn(x, y):
|
||||
return _custom_add(x, y)
|
||||
return op_fn(x, y)
|
||||
|
||||
def find_fn(target: Any, gm: fx.GraphModule):
|
||||
return gm.graph.find_nodes(op="call_function", target=target)
|
||||
@@ -155,7 +159,7 @@ class TestIrOpCustomAdd:
|
||||
torch.testing.assert_close(out_fx, out_eager)
|
||||
|
||||
# check that IR nodes only appear if enable_torch_wrap=True
|
||||
ir_nodes = find_fn(torch.ops.vllm_ir._custom_add.default, gm)
|
||||
ir_nodes = find_fn(torch_op, gm)
|
||||
if enable_torch_wrap:
|
||||
assert len(ir_nodes) == 1, gm.code
|
||||
else:
|
||||
@@ -167,7 +171,7 @@ class TestIrOpCustomAdd:
|
||||
else:
|
||||
gm = make_fx(fn)(torch.randn(2, 2), torch.randn(2, 2))
|
||||
|
||||
ir_nodes = find_fn(torch.ops.vllm_ir._custom_add.default, gm)
|
||||
ir_nodes = find_fn(torch_op, gm)
|
||||
assert len(ir_nodes) == 1, gm.code
|
||||
|
||||
|
||||
@@ -176,9 +180,12 @@ def impl_a(x: torch.Tensor, y: torch.Tensor) -> torch.Tensor:
|
||||
return x + y + 10
|
||||
|
||||
|
||||
@_custom_add.register_impl("impl_b")
|
||||
@_custom_add.register_impl("impl_b", inplace=True)
|
||||
def impl_b(x: torch.Tensor, y: torch.Tensor) -> torch.Tensor:
|
||||
return x + y + 20
|
||||
"""Computes x+y+20"""
|
||||
x.add_(y)
|
||||
x.add_(20)
|
||||
return x
|
||||
|
||||
|
||||
@_custom_add.register_impl("impl_even", supports_args=lambda x, y: x.size(1) % 2 == 0)
|
||||
@@ -243,19 +250,23 @@ class TestIrOpImplDispatch:
|
||||
# Restored to empty
|
||||
assert _custom_add.get_priority() == []
|
||||
|
||||
def test_dispatch_priority_order(self):
|
||||
@pytest.mark.parametrize("overload", ["default", "maybe_inplace"])
|
||||
def test_dispatch_priority_order(self, overload: str):
|
||||
op_fn = _custom_add if overload == "default" else _custom_add.maybe_inplace
|
||||
torch_op = getattr(torch.ops.vllm_ir._custom_add, overload)
|
||||
|
||||
x = torch.tensor(1, dtype=torch.int32)
|
||||
y = torch.tensor(2, dtype=torch.int32)
|
||||
|
||||
with _custom_add.set_priority(["impl_b", "impl_a"]):
|
||||
assert _custom_add.dispatch(x, y) is impl_b
|
||||
out1 = _custom_add(x, y)
|
||||
out2 = torch.ops.vllm_ir._custom_add(x, y)
|
||||
out1 = op_fn(x.clone(), y)
|
||||
out2 = torch_op(x.clone(), y)
|
||||
|
||||
with _custom_add.set_priority(["impl_a"]):
|
||||
assert _custom_add.dispatch(x, y) is impl_a
|
||||
out3 = _custom_add(x, y)
|
||||
out4 = torch.ops.vllm_ir._custom_add(x, y)
|
||||
out3 = op_fn(x.clone(), y)
|
||||
out4 = torch_op(x.clone(), y)
|
||||
|
||||
# impl_b
|
||||
assert out1.item() == 1 + 2 + 20
|
||||
@@ -265,18 +276,18 @@ class TestIrOpImplDispatch:
|
||||
assert out4.item() == 1 + 2 + 10
|
||||
|
||||
def test_unsupported_impl_filtered(self):
|
||||
@_custom_add.register_impl("unsupported", supported=False)
|
||||
def impl_bad(x: torch.Tensor, y: torch.Tensor) -> torch.Tensor:
|
||||
@_custom_add.register_impl("impl_unsupported", supported=False)
|
||||
def impl_unsupported(x: torch.Tensor, y: torch.Tensor) -> torch.Tensor:
|
||||
return x + y + 999
|
||||
|
||||
x = torch.tensor(1, dtype=torch.int32)
|
||||
y = torch.tensor(2, dtype=torch.int32)
|
||||
|
||||
with _custom_add.set_priority(["unsupported", "impl_a"]):
|
||||
with _custom_add.set_priority(["impl_unsupported", "impl_a"]):
|
||||
assert _custom_add.get_priority() == ["impl_a"]
|
||||
out = _custom_add(x, y)
|
||||
|
||||
# impl_bad skipped → impl_a
|
||||
# impl_unsupported skipped → impl_a
|
||||
assert out.item() == 1 + 2 + 10
|
||||
|
||||
def test_supports_args_runtime_dispatch_and_warning(
|
||||
|
||||
Reference in New Issue
Block a user