[Performance] Register Triton W4A16 GEMM as a custom op (#51453)
Signed-off-by: giuseppegrossi <ggrossi@amd.com> Co-authored-by: TJian <tunjian.tan@embeddedllm.com> Co-authored-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
co-authored by
TJian
Andreas Karatzas
parent
1d8d7a3965
commit
ee3c00bbf4
@@ -26,6 +26,7 @@ from vllm.model_executor.parameter import BasevLLMParameter, permute_param_layou
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.scalar_type import scalar_types
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.utils.torch_utils import direct_register_custom_op
|
||||
|
||||
from .MPLinearKernel import MPLinearKernel, MPLinearLayerConfig
|
||||
|
||||
@@ -161,7 +162,7 @@ def triton_w4a16_gemm_kernel(
|
||||
tl.store(c_ptrs, c, mask=mask_c)
|
||||
|
||||
|
||||
def triton_w4a16_gemm(
|
||||
def _triton_w4a16_gemm_impl(
|
||||
a: torch.Tensor, # [M, K] fp16/bf16
|
||||
b_q: torch.Tensor, # [K, N//8] int32
|
||||
scales: torch.Tensor, # [K//G, N] fp16/bf16
|
||||
@@ -270,6 +271,36 @@ def triton_w4a16_gemm(
|
||||
return c
|
||||
|
||||
|
||||
def _triton_w4a16_gemm_fake(
|
||||
a: torch.Tensor,
|
||||
b_q: torch.Tensor,
|
||||
scales: torch.Tensor,
|
||||
qzeros: torch.Tensor | None,
|
||||
group_size: int,
|
||||
zp_bias: int = 8,
|
||||
) -> torch.Tensor:
|
||||
return torch.empty((a.size(0), b_q.size(1) * 8), dtype=a.dtype, device=a.device)
|
||||
|
||||
|
||||
direct_register_custom_op(
|
||||
op_name="triton_w4a16_gemm",
|
||||
op_func=_triton_w4a16_gemm_impl,
|
||||
mutates_args=[],
|
||||
fake_impl=_triton_w4a16_gemm_fake,
|
||||
)
|
||||
|
||||
|
||||
def triton_w4a16_gemm(
|
||||
a: torch.Tensor,
|
||||
b_q: torch.Tensor,
|
||||
scales: torch.Tensor,
|
||||
qzeros: torch.Tensor | None,
|
||||
group_size: int,
|
||||
zp_bias: int = 8,
|
||||
) -> torch.Tensor:
|
||||
return torch.ops.vllm.triton_w4a16_gemm(a, b_q, scales, qzeros, group_size, zp_bias)
|
||||
|
||||
|
||||
class TritonW4A16LinearKernel(MPLinearKernel):
|
||||
"""
|
||||
Triton-based W4A16 GEMM kernel for ROCm (MI300 and newer).
|
||||
|
||||
Reference in New Issue
Block a user