[torch.compile] directly register custom op (#9896)
Signed-off-by: youkaichao <youkaichao@gmail.com>
This commit is contained in:
@@ -8,6 +8,7 @@ from vllm import _custom_ops as ops
|
||||
from vllm.model_executor.layers.fused_moe.fused_moe import (
|
||||
fused_topk, moe_align_block_size, try_get_optimal_moe_config)
|
||||
from vllm.scalar_type import scalar_types
|
||||
from vllm.utils import direct_register_custom_op
|
||||
|
||||
|
||||
def get_scalar_type(num_bits: int, has_zp: bool):
|
||||
@@ -18,7 +19,6 @@ def get_scalar_type(num_bits: int, has_zp: bool):
|
||||
return scalar_types.uint4b8 if num_bits == 4 else scalar_types.uint8b128
|
||||
|
||||
|
||||
@torch.library.custom_op("vllm::single_marlin_moe", mutates_args=[])
|
||||
def single_marlin_moe(
|
||||
hidden_states: torch.Tensor,
|
||||
w: torch.Tensor,
|
||||
@@ -119,8 +119,7 @@ def single_marlin_moe(
|
||||
return torch.sum(intermediate_cache.view(*intermediate_cache.shape), dim=1)
|
||||
|
||||
|
||||
@single_marlin_moe.register_fake
|
||||
def _(
|
||||
def single_marlin_moe_fake(
|
||||
hidden_states: torch.Tensor,
|
||||
w: torch.Tensor,
|
||||
scales: torch.Tensor,
|
||||
@@ -136,7 +135,14 @@ def _(
|
||||
return torch.empty_like(hidden_states)
|
||||
|
||||
|
||||
@torch.library.custom_op("vllm::fused_marlin_moe", mutates_args=[])
|
||||
direct_register_custom_op(
|
||||
op_name="single_marlin_moe",
|
||||
op_func=single_marlin_moe,
|
||||
mutates_args=[],
|
||||
fake_impl=single_marlin_moe_fake,
|
||||
)
|
||||
|
||||
|
||||
def fused_marlin_moe(
|
||||
hidden_states: torch.Tensor,
|
||||
w1: torch.Tensor,
|
||||
@@ -324,8 +330,7 @@ def fused_marlin_moe(
|
||||
dim=1)
|
||||
|
||||
|
||||
@fused_marlin_moe.register_fake
|
||||
def _(
|
||||
def fused_marlin_moe_fake(
|
||||
hidden_states: torch.Tensor,
|
||||
w1: torch.Tensor,
|
||||
w2: torch.Tensor,
|
||||
@@ -344,3 +349,11 @@ def _(
|
||||
is_k_full: bool = True,
|
||||
) -> torch.Tensor:
|
||||
return torch.empty_like(hidden_states)
|
||||
|
||||
|
||||
direct_register_custom_op(
|
||||
op_name="fused_marlin_moe",
|
||||
op_func=fused_marlin_moe,
|
||||
mutates_args=[],
|
||||
fake_impl=fused_marlin_moe_fake,
|
||||
)
|
||||
|
||||
@@ -12,6 +12,7 @@ import vllm.envs as envs
|
||||
from vllm import _custom_ops as ops
|
||||
from vllm.logger import init_logger
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils import direct_register_custom_op
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -466,8 +467,6 @@ def get_config_dtype_str(dtype: torch.dtype,
|
||||
return None
|
||||
|
||||
|
||||
@torch.library.custom_op("vllm::inplace_fused_experts",
|
||||
mutates_args=["hidden_states"])
|
||||
def inplace_fused_experts(hidden_states: torch.Tensor,
|
||||
w1: torch.Tensor,
|
||||
w2: torch.Tensor,
|
||||
@@ -484,22 +483,29 @@ def inplace_fused_experts(hidden_states: torch.Tensor,
|
||||
a1_scale, a2_scale)
|
||||
|
||||
|
||||
@inplace_fused_experts.register_fake
|
||||
def _(hidden_states: torch.Tensor,
|
||||
w1: torch.Tensor,
|
||||
w2: torch.Tensor,
|
||||
topk_weights: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
use_fp8_w8a8: bool = False,
|
||||
use_int8_w8a16: bool = False,
|
||||
w1_scale: Optional[torch.Tensor] = None,
|
||||
w2_scale: Optional[torch.Tensor] = None,
|
||||
a1_scale: Optional[torch.Tensor] = None,
|
||||
a2_scale: Optional[torch.Tensor] = None) -> None:
|
||||
def inplace_fused_experts_fake(
|
||||
hidden_states: torch.Tensor,
|
||||
w1: torch.Tensor,
|
||||
w2: torch.Tensor,
|
||||
topk_weights: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
use_fp8_w8a8: bool = False,
|
||||
use_int8_w8a16: bool = False,
|
||||
w1_scale: Optional[torch.Tensor] = None,
|
||||
w2_scale: Optional[torch.Tensor] = None,
|
||||
a1_scale: Optional[torch.Tensor] = None,
|
||||
a2_scale: Optional[torch.Tensor] = None) -> None:
|
||||
pass
|
||||
|
||||
|
||||
@torch.library.custom_op("vllm::outplace_fused_experts", mutates_args=[])
|
||||
direct_register_custom_op(
|
||||
op_name="inplace_fused_experts",
|
||||
op_func=inplace_fused_experts,
|
||||
mutates_args=["hidden_states"],
|
||||
fake_impl=inplace_fused_experts_fake,
|
||||
)
|
||||
|
||||
|
||||
def outplace_fused_experts(
|
||||
hidden_states: torch.Tensor,
|
||||
w1: torch.Tensor,
|
||||
@@ -517,21 +523,29 @@ def outplace_fused_experts(
|
||||
w2_scale, a1_scale, a2_scale)
|
||||
|
||||
|
||||
@outplace_fused_experts.register_fake
|
||||
def _(hidden_states: torch.Tensor,
|
||||
w1: torch.Tensor,
|
||||
w2: torch.Tensor,
|
||||
topk_weights: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
use_fp8_w8a8: bool = False,
|
||||
use_int8_w8a16: bool = False,
|
||||
w1_scale: Optional[torch.Tensor] = None,
|
||||
w2_scale: Optional[torch.Tensor] = None,
|
||||
a1_scale: Optional[torch.Tensor] = None,
|
||||
a2_scale: Optional[torch.Tensor] = None) -> torch.Tensor:
|
||||
def outplace_fused_experts_fake(
|
||||
hidden_states: torch.Tensor,
|
||||
w1: torch.Tensor,
|
||||
w2: torch.Tensor,
|
||||
topk_weights: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
use_fp8_w8a8: bool = False,
|
||||
use_int8_w8a16: bool = False,
|
||||
w1_scale: Optional[torch.Tensor] = None,
|
||||
w2_scale: Optional[torch.Tensor] = None,
|
||||
a1_scale: Optional[torch.Tensor] = None,
|
||||
a2_scale: Optional[torch.Tensor] = None) -> torch.Tensor:
|
||||
return torch.empty_like(hidden_states)
|
||||
|
||||
|
||||
direct_register_custom_op(
|
||||
op_name="outplace_fused_experts",
|
||||
op_func=outplace_fused_experts,
|
||||
mutates_args=[],
|
||||
fake_impl=outplace_fused_experts_fake,
|
||||
)
|
||||
|
||||
|
||||
def fused_experts(hidden_states: torch.Tensor,
|
||||
w1: torch.Tensor,
|
||||
w2: torch.Tensor,
|
||||
|
||||
Reference in New Issue
Block a user