[Performance] Register Triton W4A16 GEMM as a custom op (#51453)

Signed-off-by: giuseppegrossi <ggrossi@amd.com>
Co-authored-by: TJian <tunjian.tan@embeddedllm.com>
Co-authored-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
Giuseppe Grossi
2026-09-01 22:55:34 -05:00
committed by GitHub
co-authored by TJian Andreas Karatzas
parent 1d8d7a3965
commit ee3c00bbf4
2 changed files with 65 additions and 1 deletions
@@ -26,6 +26,7 @@ from vllm.model_executor.parameter import BasevLLMParameter, permute_param_layou
from vllm.platforms import current_platform
from vllm.scalar_type import scalar_types
from vllm.triton_utils import tl, triton
from vllm.utils.torch_utils import direct_register_custom_op
from .MPLinearKernel import MPLinearKernel, MPLinearLayerConfig
@@ -161,7 +162,7 @@ def triton_w4a16_gemm_kernel(
tl.store(c_ptrs, c, mask=mask_c)
def triton_w4a16_gemm(
def _triton_w4a16_gemm_impl(
a: torch.Tensor, # [M, K] fp16/bf16
b_q: torch.Tensor, # [K, N//8] int32
scales: torch.Tensor, # [K//G, N] fp16/bf16
@@ -270,6 +271,36 @@ def triton_w4a16_gemm(
return c
def _triton_w4a16_gemm_fake(
a: torch.Tensor,
b_q: torch.Tensor,
scales: torch.Tensor,
qzeros: torch.Tensor | None,
group_size: int,
zp_bias: int = 8,
) -> torch.Tensor:
return torch.empty((a.size(0), b_q.size(1) * 8), dtype=a.dtype, device=a.device)
direct_register_custom_op(
op_name="triton_w4a16_gemm",
op_func=_triton_w4a16_gemm_impl,
mutates_args=[],
fake_impl=_triton_w4a16_gemm_fake,
)
def triton_w4a16_gemm(
a: torch.Tensor,
b_q: torch.Tensor,
scales: torch.Tensor,
qzeros: torch.Tensor | None,
group_size: int,
zp_bias: int = 8,
) -> torch.Tensor:
return torch.ops.vllm.triton_w4a16_gemm(a, b_q, scales, qzeros, group_size, zp_bias)
class TritonW4A16LinearKernel(MPLinearKernel):
"""
Triton-based W4A16 GEMM kernel for ROCm (MI300 and newer).