[vLLM IR] 2/N fused_add_rms_norm and maybe_inplace overload (#36823)
Signed-off-by: Luka Govedič <lgovedic@redhat.com> Signed-off-by: Luka Govedič <ProExpertProg@users.noreply.github.com>
This commit is contained in:
@@ -7,7 +7,7 @@ import sys
|
||||
import textwrap
|
||||
import traceback
|
||||
from argparse import SUPPRESS, Action, HelpFormatter
|
||||
from collections.abc import Iterable
|
||||
from collections.abc import Callable, Iterable
|
||||
from importlib.machinery import ModuleSpec
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING, Literal
|
||||
@@ -48,6 +48,7 @@ class MockPluggableLayer:
|
||||
|
||||
|
||||
mock_if_no_torch("vllm._C", MagicMock())
|
||||
mock_if_no_torch("vllm._C_stable_libtorch", MagicMock())
|
||||
mock_if_no_torch(
|
||||
"vllm.model_executor.custom_op",
|
||||
MagicMock(CustomOp=MockCustomOp, PluggableLayer=MockPluggableLayer),
|
||||
@@ -67,6 +68,31 @@ importlib.metadata.version = lambda name: VERSIONS.get(name) or "0.0.0"
|
||||
mock_if_no_torch("torch.nn", MagicMock(Parameter=object))
|
||||
|
||||
|
||||
# Mock torch.library.infer_schema for vllm.ir.ops.IrOpInplaceOverload.__init__
|
||||
# We need to return the corresponding number of inputs, as IR infra will assert it
|
||||
def get_outputs(native_fn: Callable) -> str:
|
||||
"""
|
||||
Extract output schema from function's return type annotation,
|
||||
e.g. 'Tensor' or 'Tensor, Tensor'.
|
||||
"""
|
||||
import typing
|
||||
|
||||
return_type = typing.get_type_hints(native_fn)["return"]
|
||||
origin = typing.get_origin(return_type)
|
||||
arg_name = lambda a: a.__name__ if hasattr(a, "__name__") else str(a)
|
||||
if origin is tuple:
|
||||
args = typing.get_args(return_type)
|
||||
return ", ".join(arg_name(arg) for arg in args)
|
||||
else:
|
||||
return f"{arg_name(return_type)}"
|
||||
|
||||
|
||||
mock_if_no_torch(
|
||||
"torch.library",
|
||||
MagicMock(infer_schema=lambda fn, **k: f"(Tensor x) -> {get_outputs(fn)}"),
|
||||
)
|
||||
|
||||
|
||||
class PydanticMagicMock(MagicMock):
|
||||
"""`MagicMock` that's able to generate pydantic-core schemas."""
|
||||
|
||||
|
||||
Reference in New Issue
Block a user