Enable perf_token_group_quant/_C_stable_libtorch for ROCm (#42758)

Signed-off-by: charlifu <charlifu@amd.com>
This commit is contained in:
Charlie Fu
2026-06-02 23:23:28 -07:00
committed by GitHub
parent e0081ef8cf
commit 71df063c49
12 changed files with 145 additions and 109 deletions
@@ -158,11 +158,6 @@ class QuantFP8(CustomOp):
if use_aiter_per_token_quant:
return rocm_aiter_ops.per_token_quant(x, _FP8_DTYPE, scale)
# Fallback to native implementation for group quantization.
if self.is_group_quant:
assert scale is None, "Dynamic group quantization does not use scale"
return self._quantize_group_native(x)
# Fallback to CUDA implementation
return self.forward_cuda(x, scale, scale_ub)
@@ -575,7 +575,9 @@ def per_token_group_quant_fp8(
# prefer CUDA/XPU kernel if available
# TODO(bnell): this causes some fp8 moe test to fail.
if (current_platform.is_cuda() or current_platform.is_xpu()) and x.is_contiguous():
if (
current_platform.is_cuda_alike() or current_platform.is_xpu()
) and x.is_contiguous():
torch.ops._C.per_token_group_fp8_quant(
x,
x_q,
@@ -664,8 +666,7 @@ def per_token_group_quant_fp8_packed_for_deepgemm(
)
assert x.stride(-1) == 1, "`x` groups must be contiguous"
finfo = torch.finfo(dtype)
fp8_min, fp8_max = finfo.min, finfo.max
fp8_min, fp8_max = get_fp8_min_max()
# compute DeepGEMM-style packed scale tensor shape.
hidden_dim = x.shape[-1]
@@ -681,10 +682,10 @@ def per_token_group_quant_fp8_packed_for_deepgemm(
dtype=torch.int32,
)
# CUDA kernel path only (DeepGEMM + E8M0 is CUDA-specific).
assert current_platform.is_cuda(), (
"per_token_group_quant_fp8_packed_for_deepgemm is only valid on CUDA "
"platforms using DeepGEMM."
# Native kernel (libtorch stable); used with DeepGEMM on CUDA and
# available on ROCm for the same packed UE8M0 scale layout.
assert current_platform.is_cuda_alike(), (
"per_token_group_quant_fp8_packed_for_deepgemm requires a CUDA or ROCm GPU."
)
x_contiguous = x.contiguous()
@@ -235,8 +235,8 @@ def per_token_group_quant_int8(
device=x.device,
dtype=torch.float32,
)
# prefer CUDA kernel if available
if current_platform.is_cuda():
# Prefer native stable kernel on CUDA/ROCm when available.
if current_platform.is_cuda_alike():
torch.ops._C.per_token_group_quant_int8(
x, x_q, x_s, group_size, eps, float(int8_min), float(int8_max)
)