[Rocm][torch.compile] Adding layernorm + fp8 block quant and silu + fp8 block quant for Aiter (#25693)
Signed-off-by: charlifu <charlifu@amd.com> Signed-off-by: Micah Williamson <micah.williamson@amd.com> Signed-off-by: Charlie Fu <Charlie.Fu@amd.com> Co-authored-by: Micah Williamson <micah.williamson@amd.com> Co-authored-by: wuhuikx <hattie.wu@amd.com> Co-authored-by: Luka Govedič <ProExpertProg@users.noreply.github.com> Co-authored-by: Gregory Shtrasberg <156009573+gshtras@users.noreply.github.com>
This commit is contained in:
co-authored by
Micah Williamson
wuhuikx
Luka Govedič
Gregory Shtrasberg
parent
fccd532587
commit
3c680f4a17
@@ -196,6 +196,39 @@ direct_register_custom_op(
|
||||
)
|
||||
|
||||
|
||||
def _triton_per_token_group_quant_fp8_impl(
|
||||
x: torch.Tensor,
|
||||
group_size: int,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
return per_token_group_quant_fp8(
|
||||
x, group_size, column_major_scales=False, use_ue8m0=False
|
||||
)
|
||||
|
||||
|
||||
def _triton_per_token_group_quant_fp8_fake(
|
||||
x: torch.Tensor,
|
||||
group_size: int,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
M, N = x.shape
|
||||
x_fp8 = torch.empty((M, N), dtype=current_platform.fp8_dtype(), device=x.device)
|
||||
out_bs = torch.empty(
|
||||
(
|
||||
M,
|
||||
(N + group_size - 1) // group_size,
|
||||
),
|
||||
dtype=torch.float32,
|
||||
device=x.device,
|
||||
)
|
||||
return x_fp8, out_bs
|
||||
|
||||
|
||||
direct_register_custom_op(
|
||||
"triton_per_token_group_quant_fp8",
|
||||
_triton_per_token_group_quant_fp8_impl,
|
||||
fake_impl=_triton_per_token_group_quant_fp8_fake,
|
||||
)
|
||||
|
||||
|
||||
# TODO fix ROCm->Triton custom path:
|
||||
# https://github.com/vllm-project/vllm/issues/14397
|
||||
class W8A8BlockFp8LinearOp:
|
||||
@@ -341,17 +374,15 @@ class W8A8BlockFp8LinearOp:
|
||||
|
||||
if input_scale is not None:
|
||||
q_input = input_2d
|
||||
# MI350 case uses triton kernel
|
||||
elif use_triton:
|
||||
q_input, input_scale = per_token_group_quant_fp8(
|
||||
q_input, input_scale = torch.ops.vllm.triton_per_token_group_quant_fp8(
|
||||
input_2d,
|
||||
self.act_quant_group_shape.col,
|
||||
column_major_scales=False,
|
||||
use_ue8m0=False,
|
||||
)
|
||||
# MI300 uses tuned AITER ASM/C++ kernel
|
||||
else:
|
||||
q_input, input_scale = rocm_aiter_ops.group_fp8_quant(input_2d)
|
||||
q_input, input_scale = rocm_aiter_ops.group_fp8_quant(
|
||||
input_2d, self.act_quant_group_shape.col
|
||||
)
|
||||
|
||||
return gemm_a8w8_blockscale_op(
|
||||
q_input,
|
||||
|
||||
Reference in New Issue
Block a user