[ROCm][Quantization] add quark w4a8 mxfp4_fp8 for LinearLayer (#35316)
Signed-off-by: Divakar Verma <divakar.verma@amd.com>
This commit is contained in:
@@ -861,6 +861,39 @@ def _rocm_aiter_triton_add_rmsnorm_pad_fake(
|
||||
return out, residual_out
|
||||
|
||||
|
||||
def _rocm_aiter_gemm_a8wfp4_impl(
|
||||
x: torch.Tensor,
|
||||
w: torch.Tensor,
|
||||
x_scales: torch.Tensor,
|
||||
w_scales: torch.Tensor,
|
||||
out_dtype: torch.dtype,
|
||||
) -> torch.Tensor:
|
||||
from aiter.ops.triton.gemm_a8wfp4 import gemm_a8wfp4
|
||||
|
||||
M, N = x.shape[0], w.shape[0]
|
||||
y = torch.empty(M, N, dtype=out_dtype, device=x.device)
|
||||
gemm_a8wfp4(
|
||||
x=x,
|
||||
w=w,
|
||||
y=y,
|
||||
x_scales=x_scales,
|
||||
w_scales=w_scales,
|
||||
dtype=out_dtype,
|
||||
config=None,
|
||||
)
|
||||
return y
|
||||
|
||||
|
||||
def _rocm_aiter_gemm_a8wfp4_fake(
|
||||
x: torch.Tensor,
|
||||
w: torch.Tensor,
|
||||
x_scales: torch.Tensor,
|
||||
w_scales: torch.Tensor,
|
||||
out_dtype: torch.dtype,
|
||||
) -> torch.Tensor:
|
||||
return torch.empty(x.shape[0], w.shape[0], dtype=out_dtype, device=x.device)
|
||||
|
||||
|
||||
def _triton_rotary_embedding_impl(
|
||||
positions: torch.Tensor,
|
||||
query: torch.Tensor,
|
||||
@@ -1337,6 +1370,14 @@ class rocm_aiter_ops:
|
||||
dispatch_key=current_platform.dispatch_key,
|
||||
)
|
||||
|
||||
direct_register_custom_op(
|
||||
op_name="rocm_aiter_gemm_a8wfp4",
|
||||
op_func=_rocm_aiter_gemm_a8wfp4_impl,
|
||||
mutates_args=[],
|
||||
fake_impl=_rocm_aiter_gemm_a8wfp4_fake,
|
||||
dispatch_key=current_platform.dispatch_key,
|
||||
)
|
||||
|
||||
# Register rocm aiter rotary embedding custom op
|
||||
direct_register_custom_op(
|
||||
op_name="rocm_aiter_triton_rotary_embedding",
|
||||
@@ -1646,6 +1687,18 @@ class rocm_aiter_ops:
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
return torch.ops.vllm.rocm_aiter_per_token_quant(x, quant_dtype, scale)
|
||||
|
||||
@staticmethod
|
||||
def gemm_a8wfp4(
|
||||
x: torch.Tensor,
|
||||
w: torch.Tensor,
|
||||
x_scales: torch.Tensor,
|
||||
w_scales: torch.Tensor,
|
||||
out_dtype: torch.dtype,
|
||||
) -> torch.Tensor:
|
||||
return torch.ops.vllm.rocm_aiter_gemm_a8wfp4(
|
||||
x, w, x_scales, w_scales, out_dtype
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def triton_fp4_gemm_dynamic_qaunt(
|
||||
x: torch.Tensor,
|
||||
|
||||
Reference in New Issue
Block a user